src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

euscan: json format output

Browse Source 
Now "-f json" seems good, it supports the handler type used to retrieve
each version and outputs metadata.

Signed-off-by: volpino <fox91@anche.no>
This commit is contained in:
volpino
2012-05-23 16:30:43 +02:00
parent 8cb19b5a6b
commit a18083bd98
9 changed files with 150 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
bin/euscan
View File

@@ -212,7 +212,7 @@ def main():
# parse command line options and actions # parse command line options and actions
try: try:
packages = parse_args() queries = parse_args()
except ParseArgsException as e: except ParseArgsException as e:
if e.value == 'help': if e.value == 'help':
print_usage(help='all') print_usage(help='all')
@@ -233,11 +233,13 @@ def main():
if CONFIG['verbose'] > 2: if CONFIG['verbose'] > 2:
httplib.HTTPConnection.debuglevel = 1 httplib.HTTPConnection.debuglevel = 1
for package in packages: for query in queries:
ret = [] ret = []
output.set_query(query)
try: try:
ret = scan_upstream(package) ret = scan_upstream(query)
except AmbiguousPackageName as e: except AmbiguousPackageName as e:
pkgs = e.args[0] pkgs = e.args[0]
output.eerror("\n".join(pkgs)) output.eerror("\n".join(pkgs))
@@ -252,25 +254,28 @@ def main():
exit_helper(1) exit_helper(1)
except GentoolkitException as err: except GentoolkitException as err:
output.eerror('%s: %s' % (package, str(err))) output.eerror('%s: %s' % (query, str(err)))
exit_helper(1) exit_helper(1)
except Exception as err: except Exception as err:
output.eerror('%s: %s' % (package, str(err))) output.eerror('%s: %s' % (query, str(err)))
exit_helper(1) exit_helper(1)
if not CONFIG['quiet'] and not CONFIG['format']: if not CONFIG['quiet'] and not CONFIG['format']:
print() print()
for cp, url, version in ret: if ret is not None:
output.result(cp, version, url) if len(ret) > 0:
for cp, url, version, handler in ret:
if not len(ret) and not CONFIG['quiet']: output.result(cp, version, url, handler)
elif not CONFIG['quiet']:
output.ewarn( output.ewarn(
"Didn't find any new version, check package's homepage for " + "Didn't find any new version, check package's homepage " +
"more informations" "for more informations"
) )
output.set_query(None)
if __name__ == "__main__": if __name__ == "__main__":
setup_signals() setup_signals()

61
pym/euscan/__init__.py
View File

@@ -59,10 +59,11 @@ BRUTEFORCE_BLACKLIST_PACKAGES = [
BRUTEFORCE_BLACKLIST_URLS = [ BRUTEFORCE_BLACKLIST_URLS = [
'http://(.*)dockapps.org/download.php/id/(.*)', # infinite loop 'http://(.*)dockapps.org/download.php/id/(.*)', # infinite loop
'http://hydra.nixos.org/build/(.*)', # infinite loop 'http://hydra.nixos.org/build/(.*)', # infinite loop
'http://www.rennings.net/gentoo/distfiles/(.*)', # Doesn't respect 404, infinite loop # Doesn't respect 404, infinite loop
'http://art.gnome.org/download/(.*)', # Doesn't respect 404, infinite loop 'http://www.rennings.net/gentoo/distfiles/(.*)',
'http://barelysufficient.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop 'http://art.gnome.org/download/(.*)',
'http://olemarkus.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop 'http://barelysufficient.org/~olemarkus/(.*)',
'http://olemarkus.org/~olemarkus/(.*)',
] ]
ROBOTS_TXT_BLACKLIST_DOMAINS = [ ROBOTS_TXT_BLACKLIST_DOMAINS = [
@@ -96,26 +97,45 @@ class EuscanOutput(object):
""" """
def __init__(self, config): def __init__(self, config):
self.config = config self.config = config
self.data = defaultdict(StringIO) self.queries = defaultdict(dict)
self.packages = defaultdict(list) self.current_query = None
def set_query(self, query):
self.current_query = query
if query is not None:
if not query in self.queries:
self.queries[query] = {
"messages": defaultdict(StringIO),
"result": [],
"metadata": {},
}
def get_formatted_output(self): def get_formatted_output(self):
data = {} data = {}
for key in self.data:
if key not in ("ebegin", "eend"):
val = [x for x in self.data[key].getvalue().split("\n") if x]
data[key] = val
data["result"] = self.packages for query in self.queries:
data[query] = {
"result": self.queries[query]["result"],
"metadata": self.queries[query]["metadata"],
"messages": {}
}
for key in self.queries[query]["messages"]:
if key not in ("ebegin", "eend"):
_msg = self.queries[query]["messages"][key].getvalue()
val = [x for x in _msg.split("\n") if x]
data[query]["messages"][key] = val
if self.config["format"].lower() == "json": if self.config["format"].lower() == "json":
return json.dumps(data, indent=self.config["indent"]) return json.dumps(data, indent=self.config["indent"])
else: else:
raise TypeError("Invalid output format") raise TypeError("Invalid output format")
def result(self, cp, version, url): def result(self, cp, version, url, handler):
if self.config['format']: if self.config['format']:
self.packages[cp].append({"version": version, "url": url}) _curr = self.queries[self.current_query]
_curr["result"].append(
{"version": version, "urls": [url], "handler": handler}
)
else: else:
if not self.config['quiet']: if not self.config['quiet']:
print "Upstream Version:", pp.number("%s" % version), print "Upstream Version:", pp.number("%s" % version),
@@ -123,16 +143,21 @@ class EuscanOutput(object):
else: else:
print pp.cpv("%s-%s" % (cp, version)) + ":", pp.path(url) print pp.cpv("%s-%s" % (cp, version)) + ":", pp.path(url)
def __getattr__(self, key): def metadata(self, key, value, show=True):
output_file = self.data[key] if self.config["format"] else None if self.config["format"]:
self.queries[self.current_query]["metadata"][key] = value
elif show:
print "%s: %s" % (key.capitalize(), value)
if output_file: def __getattr__(self, key):
_output = EOutputFile(out_file=self.data[key], if self.config["format"]:
out_file = self.queries[self.current_query]["messages"][key]
_output = EOutputFile(out_file=out_file,
quiet=self.config['quiet']) quiet=self.config['quiet'])
ret = getattr(_output, key) ret = getattr(_output, key)
else: else:
ret = getattr(EOutputFile(quiet=self.config['quiet']), key) ret = getattr(EOutputFile(quiet=self.config['quiet']), key)
return ret return ret

9
pym/euscan/handlers/cpan.py
View File

@@ -3,8 +3,9 @@ import portage
import urllib2 import urllib2
import json import json
from euscan import helpers from euscan import helpers, output
import euscan
HANDLER_NAME = "cpan"
_cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*") _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*")
@@ -83,7 +84,7 @@ def scan(cpv, url):
orig_url = url orig_url = url
url = 'http://search.cpan.org/api/dist/%s' % pkg url = 'http://search.cpan.org/api/dist/%s' % pkg
euscan.output.einfo("Using: " + url) output.einfo("Using: " + url)
try: try:
fp = helpers.urlopen(url) fp = helpers.urlopen(url)
@@ -125,7 +126,7 @@ def scan(cpv, url):
if url == orig_url: if url == orig_url:
continue continue
ret.append((url, pv)) ret.append((url, pv, HANDLER_NAME))
return ret return ret

39
pym/euscan/handlers/generic.py
View File

@@ -7,9 +7,10 @@ from BeautifulSoup import BeautifulSoup
import portage import portage
from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \ from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \
BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS, output, helpers
from euscan import helpers
import euscan HANDLER_NAME = "generic"
BRUTEFORCE_HANDLER_NAME = "brute_force"
def scan_html(data, url, pattern): def scan_html(data, url, pattern):
@@ -53,7 +54,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
steps = steps[1:] steps = steps[1:]
euscan.output.einfo("Scanning: %s" % url) output.einfo("Scanning: %s" % url)
try: try:
fp = helpers.urlopen(url) fp = helpers.urlopen(url)
@@ -87,7 +88,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
path = url + path path = url + path
if not steps and path not in orig_url: if not steps and path not in orig_url:
versions.append((path, pv)) versions.append((path, pv, HANDLER_NAME))
if steps: if steps:
ret = scan_directory_recursive(cp, ver, rev, path, steps, orig_url) ret = scan_directory_recursive(cp, ver, rev, path, steps, orig_url)
@@ -99,7 +100,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
def scan(cpv, url): def scan(cpv, url):
for bu in SCANDIR_BLACKLIST_URLS: for bu in SCANDIR_BLACKLIST_URLS:
if re.match(bu, url): if re.match(bu, url):
euscan.output.einfo("%s is blacklisted by rule %s" % (url, bu)) output.einfo("%s is blacklisted by rule %s" % (url, bu))
return [] return []
resolved_url = helpers.parse_mirror(url) resolved_url = helpers.parse_mirror(url)
@@ -112,23 +113,25 @@ def scan(cpv, url):
if ver not in resolved_url: if ver not in resolved_url:
newver = helpers.version_change_end_sep(ver) newver = helpers.version_change_end_sep(ver)
if newver and newver in resolved_url: if newver and newver in resolved_url:
euscan.output.einfo( output.einfo(
"Version: using %s instead of %s" % (newver, ver) "Version: using %s instead of %s" % (newver, ver)
) )
ver = newver ver = newver
template = helpers.template_from_url(resolved_url, ver) template = helpers.template_from_url(resolved_url, ver)
if '${' not in template: if '${' not in template:
euscan.output.einfo( output.einfo(
"Url doesn't seems to depend on version: %s not found in %s" % "Url doesn't seems to depend on version: %s not found in %s" %
(ver, resolved_url) (ver, resolved_url)
) )
return [] return []
else: else:
euscan.output.einfo("Scanning: %s" % template) output.einfo("Scanning: %s" % template)
steps = helpers.generate_scan_paths(template) steps = helpers.generate_scan_paths(template)
return scan_directory_recursive(cp, ver, rev, "", steps, url) ret = scan_directory_recursive(cp, ver, rev, "", steps, url)
return ret
def brute_force(cpv, url): def brute_force(cpv, url):
@@ -140,15 +143,15 @@ def brute_force(cpv, url):
for bp in BRUTEFORCE_BLACKLIST_PACKAGES: for bp in BRUTEFORCE_BLACKLIST_PACKAGES:
if re.match(bp, cp): if re.match(bp, cp):
euscan.output.einfo("%s is blacklisted by rule %s" % (cp, bp)) output.einfo("%s is blacklisted by rule %s" % (cp, bp))
return [] return []
for bp in BRUTEFORCE_BLACKLIST_URLS: for bp in BRUTEFORCE_BLACKLIST_URLS:
if re.match(bp, url): if re.match(bp, url):
euscan.output.einfo("%s is blacklisted by rule %s" % (cp, bp)) output.einfo("%s is blacklisted by rule %s" % (cp, bp))
return [] return []
euscan.output.einfo("Generating version from " + ver) output.einfo("Generating version from " + ver)
components = helpers.split_version(ver) components = helpers.split_version(ver)
versions = helpers.gen_versions(components, CONFIG["brute-force"]) versions = helpers.gen_versions(components, CONFIG["brute-force"])
@@ -159,18 +162,18 @@ def brute_force(cpv, url):
versions.remove(v) versions.remove(v)
if not versions: if not versions:
euscan.output.einfo("Can't generate new versions from " + ver) output.einfo("Can't generate new versions from " + ver)
return [] return []
template = helpers.template_from_url(url, ver) template = helpers.template_from_url(url, ver)
if '${PV}' not in template: if '${PV}' not in template:
euscan.output.einfo( output.einfo(
"Url doesn't seems to depend on full version: %s not found in %s" % "Url doesn't seems to depend on full version: %s not found in %s" %
(ver, url)) (ver, url))
return [] return []
else: else:
euscan.output.einfo("Brute forcing: %s" % template) output.einfo("Brute forcing: %s" % template)
result = [] result = []
@@ -195,10 +198,10 @@ def brute_force(cpv, url):
if not infos: if not infos:
continue continue
result.append([url, version]) result.append([url, version, BRUTEFORCE_HANDLER_NAME])
if len(result) > CONFIG['brute-force-false-watermark']: if len(result) > CONFIG['brute-force-false-watermark']:
euscan.output.einfo( output.einfo(
"Broken server detected ! Skipping brute force." "Broken server detected ! Skipping brute force."
) )
return [] return []

4
pym/euscan/handlers/kde.py
View File

@@ -1,5 +1,7 @@
from euscan.handlers import generic from euscan.handlers import generic
HANDLER_NAME = "kde"
def can_handle(cpv, url): def can_handle(cpv, url):
if url.startswith('mirror://kde/'): if url.startswith('mirror://kde/'):
@@ -13,7 +15,7 @@ def clean_results(results):
for path, version in results: for path, version in results:
if version == '5SUMS': if version == '5SUMS':
continue continue
ret.append((path, version)) ret.append((path, version, HANDLER_NAME))
return ret return ret

9
pym/euscan/handlers/php.py
View File

@@ -3,8 +3,9 @@ import portage
import urllib2 import urllib2
import xml.dom.minidom import xml.dom.minidom
from euscan import helpers from euscan import helpers, output
import euscan
HANDLER_NAME = "php"
def can_handle(cpv, url): def can_handle(cpv, url):
@@ -34,7 +35,7 @@ def scan(cpv, url):
orig_url = url orig_url = url
url = 'http://%s/rest/r/%s/allreleases.xml' % (channel, pkg.lower()) url = 'http://%s/rest/r/%s/allreleases.xml' % (channel, pkg.lower())
euscan.output.einfo("Using: " + url) output.einfo("Using: " + url)
try: try:
fp = helpers.urlopen(url) fp = helpers.urlopen(url)
@@ -64,7 +65,7 @@ def scan(cpv, url):
if url == orig_url: if url == orig_url:
continue continue
ret.append((url, pv)) ret.append((url, pv, HANDLER_NAME))
return ret return ret

9
pym/euscan/handlers/pypi.py
View File

@@ -3,8 +3,9 @@ import re
import portage import portage
from euscan import helpers from euscan import helpers, output
import euscan
HANDLER_NAME = "pypi"
def can_handle(cpv, url): def can_handle(cpv, url):
@@ -26,7 +27,7 @@ def scan(cpv, url):
package = guess_package(cpv, url) package = guess_package(cpv, url)
euscan.output.einfo("Using PyPi XMLRPC: " + package) output.einfo("Using PyPi XMLRPC: " + package)
client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi') client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
versions = client.package_releases(package) versions = client.package_releases(package)
@@ -46,7 +47,7 @@ def scan(cpv, url):
continue continue
urls = client.release_urls(package, up_pv) urls = client.release_urls(package, up_pv)
urls = " ".join([infos['url'] for infos in urls]) urls = " ".join([infos['url'] for infos in urls])
ret.append((urls, pv)) ret.append((urls, pv, HANDLER_NAME))
return ret return ret

11
pym/euscan/handlers/rubygem.py
View File

@@ -3,8 +3,9 @@ import portage
import json import json
import urllib2 import urllib2
from euscan import helpers from euscan import helpers, output
import euscan
HANDLER_NAME = "rubygem"
def can_handle(cpv, url): def can_handle(cpv, url):
@@ -31,13 +32,13 @@ def scan(cpv, url):
gem = guess_gem(cpv, url) gem = guess_gem(cpv, url)
if not gem: if not gem:
euscan.output.eerror("Can't guess gem name using %s and %s" % \ output.eerror("Can't guess gem name using %s and %s" % \
(cpv, url)) (cpv, url))
return [] return []
url = 'http://rubygems.org/api/v1/versions/%s.json' % gem url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
euscan.output.einfo("Using: " + url) output.einfo("Using: " + url)
try: try:
fp = helpers.urlopen(url) fp = helpers.urlopen(url)
@@ -65,7 +66,7 @@ def scan(cpv, url):
if helpers.version_filtered(cp, ver, pv): if helpers.version_filtered(cp, ver, pv):
continue continue
url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv) url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
ret.append((url, pv)) ret.append((url, pv, HANDLER_NAME))
return ret return ret

71
pym/euscan/scan.py
View File

@@ -1,8 +1,7 @@
import os import os
import sys from datetime import datetime
import portage import portage
from portage.dbapi import porttree from portage.dbapi import porttree
import gentoolkit.pprinter as pp import gentoolkit.pprinter as pp
@@ -10,28 +9,28 @@ from gentoolkit.query import Query
from gentoolkit.eclean.search import (port_settings) from gentoolkit.eclean.search import (port_settings)
from euscan import CONFIG, BLACKLIST_PACKAGES from euscan import CONFIG, BLACKLIST_PACKAGES
from euscan import handlers from euscan import handlers, helpers, output
from euscan import helpers
import euscan
def filter_versions(cp, versions): def filter_versions(cp, versions):
filtered = {} filtered = {}
for url, version in versions: for url, version, handler in versions:
''' Try to keep the most specific urls (determinted by the length) ''' # Try to keep the most specific urls (determinted by the length)
if version in filtered and len(url) < len(filtered[version]): if version in filtered and len(url) < len(filtered[version]):
continue continue
''' Remove blacklisted versions ''' # Remove blacklisted versions
if helpers.version_blacklisted(cp, version): if helpers.version_blacklisted(cp, version):
continue continue
filtered[version] = url filtered[version] = {"url": url, "handler": handler}
return [(cp, filtered[version], version) for version in filtered] return [
(cp, filtered[version]["url"], version, filtered[version]["handler"])
for version in filtered
]
def scan_upstream_urls(cpv, urls): def scan_upstream_urls(cpv, urls):
@@ -41,20 +40,20 @@ def scan_upstream_urls(cpv, urls):
for url in urls[filename]: for url in urls[filename]:
if not CONFIG['quiet'] and not CONFIG['format']: if not CONFIG['quiet'] and not CONFIG['format']:
pp.uprint() pp.uprint()
euscan.output.einfo("SRC_URI is '%s'" % url) output.einfo("SRC_URI is '%s'" % url)
if '://' not in url: if '://' not in url:
euscan.output.einfo("Invalid url '%s'" % url) output.einfo("Invalid url '%s'" % url)
continue continue
''' Try normal scan ''' # Try normal scan
if CONFIG["scan-dir"]: if CONFIG["scan-dir"]:
versions.extend(handlers.scan(cpv, url)) versions.extend(handlers.scan(cpv, url))
if versions and CONFIG['oneshot']: if versions and CONFIG['oneshot']:
break break
''' Brute Force ''' # Brute Force
if CONFIG["brute-force"] > 0: if CONFIG["brute-force"] > 0:
versions.extend(handlers.brute_force(cpv, url)) versions.extend(handlers.brute_force(cpv, url))
@@ -79,10 +78,10 @@ def scan_upstream(query):
) )
if not matches: if not matches:
sys.stderr.write( output.ewarn(
pp.warn("No package matching '%s'" % pp.pkgquery(query)) pp.warn("No package matching '%s'" % pp.pkgquery(query))
) )
return [] return None
matches = sorted(matches) matches = sorted(matches)
pkg = matches.pop() pkg = matches.pop()
@@ -91,29 +90,41 @@ def scan_upstream(query):
pkg = matches.pop() pkg = matches.pop()
if not pkg: if not pkg:
sys.stderr.write(pp.warn("Package '%s' only have a dev version (9999)" output.ewarn(
% pp.pkgquery(pkg.cp))) pp.warn("Package '%s' only have a dev version (9999)"
return [] % pp.pkgquery(pkg.cp))
)
return None
# useful data only for formatted output
output.metadata("datetime", datetime.now().isoformat(), show=False)
output.metadata("cp", pkg.cp, show=False)
output.metadata("cpv", pkg.cpv, show=False)
if pkg.cp in BLACKLIST_PACKAGES: if pkg.cp in BLACKLIST_PACKAGES:
sys.stderr.write( output.ewarn(
pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp)) pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp))
) )
return [] return None
if not CONFIG['quiet'] and not CONFIG['format']: if not CONFIG['quiet']:
if not CONFIG['format']:
pp.uprint( pp.uprint(
" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name())) " * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name()))
) )
pp.uprint() pp.uprint()
else:
output.metadata("overlay", pp.section(pkg.repo_name()))
ebuild_path = pkg.ebuild_path() ebuild_path = pkg.ebuild_path()
if ebuild_path: if ebuild_path:
pp.uprint('Ebuild: ' + pp.path(os.path.normpath(ebuild_path))) output.metadata(
"ebuild", pp.path(os.path.normpath(ebuild_path))
)
pp.uprint('Repository: ' + pkg.repo_name()) output.metadata("repository", pkg.repo_name())
pp.uprint('Homepage: ' + pkg.environment("HOMEPAGE")) output.metadata("homepage", pkg.environment("HOMEPAGE"))
pp.uprint('Description: ' + pkg.environment("DESCRIPTION")) output.metadata("description", pkg.environment("DESCRIPTION"))
cpv = pkg.cpv cpv = pkg.cpv
metadata = { metadata = {
@@ -125,11 +136,11 @@ def scan_upstream(query):
alist = porttree._parse_uri_map(cpv, metadata, use=use) alist = porttree._parse_uri_map(cpv, metadata, use=use)
aalist = porttree._parse_uri_map(cpv, metadata) aalist = porttree._parse_uri_map(cpv, metadata)
except Exception as e: except Exception as e:
sys.stderr.write(pp.warn("%s\n" % str(e))) output.ewarn(pp.warn("%s\n" % str(e)))
sys.stderr.write( output.ewarn(
pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv)) pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv))
) )
return [] return None
if "mirror" in portage.settings.features: if "mirror" in portage.settings.features:
urls = aalist urls = aalist