Merge git://github.com/volpino/euscan
Conflicts: pym/euscan/scan.py
This commit is contained in:
Corentin Chary
commit
687851ffcb
178
bin/euscan
178
bin/euscan
@ -6,36 +6,43 @@ Distributed under the terms of the GNU General Public License v2
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
""" Meta """
|
||||
|
||||
# Meta
|
||||
|
||||
__author__ = "Corentin Chary (iksaif)"
|
||||
__email__ = "corentin.chary@gmail.com"
|
||||
__version__ = "git"
|
||||
__productname__ = "euscan"
|
||||
__description__ = "A tool to detect new upstream releases."
|
||||
__version__ = "git"
|
||||
|
||||
""" Imports """
|
||||
|
||||
# Imports
|
||||
|
||||
import sys
|
||||
import getopt
|
||||
import errno
|
||||
import httplib
|
||||
|
||||
from portage.output import white, yellow, turquoise, green, EOutput
|
||||
from portage.output import white, yellow, turquoise, green
|
||||
from portage.exception import AmbiguousPackageName
|
||||
|
||||
from gentoolkit import pprinter as pp
|
||||
from gentoolkit.eclean.search import (port_settings)
|
||||
from gentoolkit.errors import GentoolkitException
|
||||
|
||||
import euscan
|
||||
from euscan import CONFIG
|
||||
from euscan import CONFIG, output
|
||||
from euscan.scan import scan_upstream
|
||||
|
||||
""" Globals """
|
||||
|
||||
# Globals
|
||||
|
||||
def exit_helper(status):
|
||||
if CONFIG["format"]:
|
||||
print(output.get_formatted_output())
|
||||
sys.exit(status)
|
||||
|
||||
|
||||
def setupSignals():
|
||||
def setup_signals():
|
||||
"""This block ensures that ^C interrupts are handled quietly."""
|
||||
import signal
|
||||
|
||||
@ -43,14 +50,14 @@ def setupSignals():
|
||||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
print()
|
||||
sys.exit(errno.EINTR)
|
||||
exit_helper(errno.EINTR)
|
||||
|
||||
signal.signal(signal.SIGINT, exithandler)
|
||||
signal.signal(signal.SIGTERM, exithandler)
|
||||
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
|
||||
|
||||
|
||||
def printVersion():
|
||||
def print_version():
|
||||
"""Output the version info."""
|
||||
print("%s (%s) - %s" \
|
||||
% (__productname__, __version__, __description__))
|
||||
@ -60,23 +67,26 @@ def printVersion():
|
||||
print("Distributed under the terms of the GNU General Public License v2")
|
||||
|
||||
|
||||
def printUsage(_error=None, help=None):
|
||||
def print_usage(_error=None, help=None):
|
||||
"""Print help message. May also print partial help to stderr if an
|
||||
error from {'options'} is specified."""
|
||||
|
||||
out = sys.stdout
|
||||
if _error:
|
||||
out = sys.stderr
|
||||
|
||||
if not _error in ('global-options', 'packages',):
|
||||
_error = None
|
||||
|
||||
if not _error and not help:
|
||||
help = 'all'
|
||||
|
||||
if _error in ('global-options',):
|
||||
print(pp.error("Wrong option on command line."), file=out)
|
||||
print(file=out)
|
||||
output.eerror("Wrong option on command line.\n")
|
||||
|
||||
if _error in ('packages',):
|
||||
print(pp.error("You need to specify exactly one package."), file=out)
|
||||
print(file=out)
|
||||
output.eerror("You need to specify exactly one package.\n")
|
||||
|
||||
print(white("Usage:"), file=out)
|
||||
if _error in ('global-options', 'packages',) or help == 'all':
|
||||
print(" " + turquoise(__productname__),
|
||||
@ -106,14 +116,19 @@ def printUsage(_error=None, help=None):
|
||||
" (default: 2)\n" +
|
||||
" " * 29 + "bigger levels will generate more versions numbers\n" +
|
||||
" " * 29 + "0 means disabled", file=out)
|
||||
print(yellow(" -f, --format=<format>") +
|
||||
" - define the output " + yellow("<format>") +
|
||||
" (available: json)", file=out)
|
||||
print(file=out)
|
||||
|
||||
if _error in ('packages',) or help:
|
||||
print(green(" package") +
|
||||
" - the packages (or ebuilds) you want to scan",
|
||||
file=out)
|
||||
print(file=out)
|
||||
'''print( "More detailed instruction can be found in",
|
||||
turquoise("`man %s`" % __productname__), file=out)'''
|
||||
|
||||
#print( "More detailed instruction can be found in",
|
||||
#turquoise("`man %s`" % __productname__), file=out)
|
||||
|
||||
|
||||
class ParseArgsException(Exception):
|
||||
@ -125,12 +140,12 @@ class ParseArgsException(Exception):
|
||||
return repr(self.value)
|
||||
|
||||
|
||||
def parseArgs():
|
||||
def parse_args():
|
||||
"""Parse the command line arguments. Raise exceptions on
|
||||
errors. Returns package and affect the CONFIG dict.
|
||||
errors. Returns packages and affects the CONFIG dict.
|
||||
"""
|
||||
|
||||
def optionSwitch(opts):
|
||||
def option_switch(opts):
|
||||
"""local function for interpreting command line options
|
||||
and setting options accordingly"""
|
||||
return_code = True
|
||||
@ -151,29 +166,35 @@ def parseArgs():
|
||||
CONFIG['brute-force'] = int(a)
|
||||
elif o in ("-v", "--verbose") and not CONFIG['quiet']:
|
||||
CONFIG['verbose'] += 1
|
||||
elif o in ("-f", "--format"):
|
||||
CONFIG['format'] = a
|
||||
CONFIG['nocolor'] = True
|
||||
pp.output.nocolor()
|
||||
else:
|
||||
return_code = False
|
||||
|
||||
return return_code
|
||||
|
||||
' here are the different allowed command line options (getopt args) '
|
||||
# here are the different allowed command line options (getopt args)
|
||||
getopt_options = {'short': {}, 'long': {}}
|
||||
getopt_options['short']['global'] = "hVCqv1b:"
|
||||
getopt_options['long']['global'] = ["help", "version", "nocolor", "quiet",
|
||||
"verbose", "oneshot", "brute-force="]
|
||||
getopt_options['short']['global'] = "hVCqv1bf:"
|
||||
getopt_options['long']['global'] = [
|
||||
"help", "version", "nocolor", "quiet", "verbose", "oneshot",
|
||||
"brute-force=", "format="
|
||||
]
|
||||
|
||||
short_opts = getopt_options['short']['global']
|
||||
long_opts = getopt_options['long']['global']
|
||||
opts_mode = 'global'
|
||||
|
||||
' apply getopts to command line, show partial help on failure '
|
||||
# apply getopts to command line, show partial help on failure
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts)
|
||||
except:
|
||||
raise ParseArgsException(opts_mode + '-options')
|
||||
|
||||
' set options accordingly '
|
||||
optionSwitch(opts)
|
||||
# set options accordingly
|
||||
option_switch(opts)
|
||||
|
||||
if len(args) < 1:
|
||||
raise ParseArgsException('packages')
|
||||
@ -183,81 +204,80 @@ def parseArgs():
|
||||
|
||||
def main():
|
||||
"""Parse command line and execute all actions."""
|
||||
CONFIG['nocolor'] = (port_settings["NOCOLOR"] in ('yes', 'true')
|
||||
or not sys.stdout.isatty())
|
||||
CONFIG['nocolor'] = (
|
||||
port_settings["NOCOLOR"] in ('yes', 'true') or not sys.stdout.isatty()
|
||||
)
|
||||
if CONFIG['nocolor']:
|
||||
pp.output.nocolor()
|
||||
' parse command line options and actions '
|
||||
|
||||
# parse command line options and actions
|
||||
try:
|
||||
packages = parseArgs()
|
||||
queries = parse_args()
|
||||
except ParseArgsException as e:
|
||||
if e.value == 'help':
|
||||
printUsage(help='all')
|
||||
sys.exit(0)
|
||||
elif e.value[:5] == 'help-':
|
||||
printUsage(help=e.value[5:])
|
||||
sys.exit(0)
|
||||
elif e.value == 'version':
|
||||
printVersion()
|
||||
sys.exit(0)
|
||||
else:
|
||||
printUsage(e.value)
|
||||
sys.exit(errno.EINVAL)
|
||||
print_usage(help='all')
|
||||
exit_helper(0)
|
||||
|
||||
elif e.value[:5] == 'help-':
|
||||
print_usage(help=e.value[5:])
|
||||
exit_helper(0)
|
||||
|
||||
elif e.value == 'version':
|
||||
print_version()
|
||||
exit_helper(0)
|
||||
|
||||
else:
|
||||
print_usage(e.value)
|
||||
exit_helper(errno.EINVAL)
|
||||
|
||||
""" Change euscan's output """
|
||||
euscan.output = EOutput(CONFIG['quiet'])
|
||||
if CONFIG['verbose'] > 2:
|
||||
httplib.HTTPConnection.debuglevel = 1
|
||||
|
||||
for package in packages:
|
||||
for query in queries:
|
||||
ret = []
|
||||
|
||||
output.set_query(query)
|
||||
|
||||
try:
|
||||
ret = scan_upstream(package)
|
||||
ret = scan_upstream(query)
|
||||
except AmbiguousPackageName as e:
|
||||
pkgs = e.args[0]
|
||||
for candidate in pkgs:
|
||||
print(candidate)
|
||||
output.eerror("\n".join(pkgs))
|
||||
|
||||
from os.path import basename # To get the short name
|
||||
|
||||
print(file=sys.stderr)
|
||||
print(
|
||||
pp.error(
|
||||
"The short ebuild name '%s' is ambiguous. Please specify" \
|
||||
% basename(pkgs[0])
|
||||
),
|
||||
file=sys.stderr, end=""
|
||||
output.error(
|
||||
"The short ebuild name '%s' is ambiguous. Please specify" %
|
||||
basename(pkgs[0]),
|
||||
"one of the above fully-qualified ebuild names instead."
|
||||
)
|
||||
pp.die(1, "one of the above fully-qualified ebuild names instead.")
|
||||
except GentoolkitException as err:
|
||||
pp.die(1, '%s: %s' % (package, str(err)))
|
||||
except Exception as err:
|
||||
pp.die(1, '%s: %s' % (package, str(err)))
|
||||
exit_helper(1)
|
||||
|
||||
if not CONFIG['quiet']:
|
||||
except GentoolkitException as err:
|
||||
output.eerror('%s: %s' % (query, str(err)))
|
||||
exit_helper(1)
|
||||
|
||||
except Exception as err:
|
||||
output.eerror('%s: %s' % (query, str(err)))
|
||||
exit_helper(1)
|
||||
|
||||
if not CONFIG['quiet'] and not CONFIG['format']:
|
||||
print()
|
||||
|
||||
for cp, url, version in ret:
|
||||
if not CONFIG['quiet']:
|
||||
print("Upstream Version: "
|
||||
+ pp.number("%s" % version)
|
||||
+ pp.path(" %s" % url))
|
||||
else:
|
||||
print(pp.cpv("%s-%s" % (cp, version))
|
||||
+ ": " + pp.path(url))
|
||||
if ret is not None:
|
||||
if len(ret) > 0:
|
||||
for cp, url, version, handler, confidence in ret:
|
||||
output.result(cp, version, url, handler, confidence)
|
||||
elif not CONFIG['quiet']:
|
||||
output.ewarn(
|
||||
"Didn't find any new version, check package's homepage " +
|
||||
"for more informations"
|
||||
)
|
||||
|
||||
if not len(ret) and not CONFIG['quiet']:
|
||||
print(pp.warn("Didn't find any new version, "
|
||||
+ "check package's homepage for "
|
||||
+ "more informations"))
|
||||
output.set_query(None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
setupSignals()
|
||||
setup_signals()
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print("Aborted.")
|
||||
sys.exit(errno.EINTR)
|
||||
sys.exit(0)
|
||||
exit_helper(0)
|
||||
|
||||
@ -3,11 +3,17 @@
|
||||
# Copyright 2011 Corentin Chary <corentin.chary@gmail.com>
|
||||
# Distributed under the terms of the GNU General Public License v2
|
||||
|
||||
from io import StringIO
|
||||
from collections import defaultdict
|
||||
import json
|
||||
|
||||
from gentoolkit import pprinter as pp
|
||||
from portage.output import EOutput
|
||||
|
||||
|
||||
__version__ = "git"
|
||||
|
||||
|
||||
from portage.output import EOutput
|
||||
|
||||
CONFIG = {
|
||||
'nocolor': False,
|
||||
'quiet': False,
|
||||
@ -20,11 +26,11 @@ CONFIG = {
|
||||
'oneshot': True,
|
||||
'user-agent': 'escan (http://euscan.iksaif.net)',
|
||||
'skip-robots-txt': False,
|
||||
'cache': False
|
||||
'cache': False,
|
||||
'format': None,
|
||||
'indent': 2
|
||||
}
|
||||
|
||||
output = EOutput(CONFIG['quiet'])
|
||||
|
||||
BLACKLIST_VERSIONS = [
|
||||
# Compatibility package for running binaries linked against a
|
||||
# pre gcc 3.4 libstdc++, won't be updated
|
||||
@ -53,10 +59,11 @@ BRUTEFORCE_BLACKLIST_PACKAGES = [
|
||||
BRUTEFORCE_BLACKLIST_URLS = [
|
||||
'http://(.*)dockapps.org/download.php/id/(.*)', # infinite loop
|
||||
'http://hydra.nixos.org/build/(.*)', # infinite loop
|
||||
'http://www.rennings.net/gentoo/distfiles/(.*)', # Doesn't respect 404, infinite loop
|
||||
'http://art.gnome.org/download/(.*)', # Doesn't respect 404, infinite loop
|
||||
'http://barelysufficient.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop
|
||||
'http://olemarkus.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop
|
||||
# Doesn't respect 404, infinite loop
|
||||
'http://www.rennings.net/gentoo/distfiles/(.*)',
|
||||
'http://art.gnome.org/download/(.*)',
|
||||
'http://barelysufficient.org/~olemarkus/(.*)',
|
||||
'http://olemarkus.org/~olemarkus/(.*)',
|
||||
]
|
||||
|
||||
ROBOTS_TXT_BLACKLIST_DOMAINS = [
|
||||
@ -67,3 +74,94 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [
|
||||
'(.*)chromium.org(.*)',
|
||||
'(.*)nodejs.org(.*)',
|
||||
]
|
||||
|
||||
|
||||
class EOutputFile(EOutput):
|
||||
"""
|
||||
Override of EOutput, allows to specify an output file for writes
|
||||
"""
|
||||
def __init__(self, out_file=None, *args, **kwargs):
|
||||
super(EOutputFile, self).__init__(*args, **kwargs)
|
||||
self.out_file = out_file
|
||||
|
||||
def _write(self, f, msg):
|
||||
if self.out_file is None:
|
||||
super(EOutputFile, self)._write(f, msg)
|
||||
else:
|
||||
super(EOutputFile, self)._write(self.out_file, msg)
|
||||
|
||||
|
||||
class EuscanOutput(object):
|
||||
"""
|
||||
Class that handles output for euscan
|
||||
"""
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.queries = defaultdict(dict)
|
||||
self.current_query = None
|
||||
|
||||
def set_query(self, query):
|
||||
self.current_query = query
|
||||
if query is not None:
|
||||
if not query in self.queries:
|
||||
self.queries[query] = {
|
||||
"messages": defaultdict(StringIO),
|
||||
"result": [],
|
||||
"metadata": {},
|
||||
}
|
||||
|
||||
def get_formatted_output(self):
|
||||
data = {}
|
||||
|
||||
for query in self.queries:
|
||||
data[query] = {
|
||||
"result": self.queries[query]["result"],
|
||||
"metadata": self.queries[query]["metadata"],
|
||||
"messages": {}
|
||||
}
|
||||
for key in self.queries[query]["messages"]:
|
||||
if key not in ("ebegin", "eend"):
|
||||
_msg = self.queries[query]["messages"][key].getvalue()
|
||||
val = [x for x in _msg.split("\n") if x]
|
||||
data[query]["messages"][key] = val
|
||||
|
||||
if self.config["format"].lower() == "json":
|
||||
return json.dumps(data, indent=self.config["indent"])
|
||||
else:
|
||||
raise TypeError("Invalid output format")
|
||||
|
||||
def result(self, cp, version, url, handler, confidence):
|
||||
from euscan.helpers import get_version_type
|
||||
|
||||
if self.config['format']:
|
||||
_curr = self.queries[self.current_query]
|
||||
_curr["result"].append(
|
||||
{"version": version, "urls": [url], "handler": handler,
|
||||
"confidence": confidence, "type": get_version_type(version)}
|
||||
)
|
||||
else:
|
||||
if not self.config['quiet']:
|
||||
print "Upstream Version:", pp.number("%s" % version),
|
||||
print pp.path(" %s" % url)
|
||||
else:
|
||||
print pp.cpv("%s-%s" % (cp, version)) + ":", pp.path(url)
|
||||
|
||||
def metadata(self, key, value, show=True):
|
||||
if self.config["format"]:
|
||||
self.queries[self.current_query]["metadata"][key] = value
|
||||
elif show:
|
||||
print "%s: %s" % (key.capitalize(), value)
|
||||
|
||||
def __getattr__(self, key):
|
||||
if self.config["format"]:
|
||||
out_file = self.queries[self.current_query]["messages"][key]
|
||||
|
||||
_output = EOutputFile(out_file=out_file,
|
||||
quiet=self.config['quiet'])
|
||||
ret = getattr(_output, key)
|
||||
else:
|
||||
ret = getattr(EOutputFile(quiet=self.config['quiet']), key)
|
||||
return ret
|
||||
|
||||
|
||||
output = EuscanOutput(CONFIG)
|
||||
|
||||
@ -3,8 +3,10 @@ import portage
|
||||
import urllib2
|
||||
import json
|
||||
|
||||
from euscan import helpers
|
||||
import euscan
|
||||
from euscan import helpers, output
|
||||
|
||||
HANDLER_NAME = "cpan"
|
||||
CONFIDENCE = 100.0
|
||||
|
||||
_cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*")
|
||||
|
||||
@ -83,7 +85,7 @@ def scan(cpv, url):
|
||||
orig_url = url
|
||||
url = 'http://search.cpan.org/api/dist/%s' % pkg
|
||||
|
||||
euscan.output.einfo("Using: " + url)
|
||||
output.einfo("Using: " + url)
|
||||
|
||||
try:
|
||||
fp = helpers.urlopen(url)
|
||||
@ -125,7 +127,7 @@ def scan(cpv, url):
|
||||
if url == orig_url:
|
||||
continue
|
||||
|
||||
ret.append((url, pv))
|
||||
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
@ -7,9 +7,13 @@ from BeautifulSoup import BeautifulSoup
|
||||
import portage
|
||||
|
||||
from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \
|
||||
BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS
|
||||
from euscan import helpers
|
||||
import euscan
|
||||
BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS, output, helpers
|
||||
|
||||
HANDLER_NAME = "generic"
|
||||
CONFIDENCE = 50.0
|
||||
|
||||
BRUTEFORCE_HANDLER_NAME = "brute_force"
|
||||
BRUTEFORCE_CONFIDENCE = 30.0
|
||||
|
||||
|
||||
def scan_html(data, url, pattern):
|
||||
@ -53,7 +57,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
|
||||
|
||||
steps = steps[1:]
|
||||
|
||||
euscan.output.einfo("Scanning: %s" % url)
|
||||
output.einfo("Scanning: %s" % url)
|
||||
|
||||
try:
|
||||
fp = helpers.urlopen(url)
|
||||
@ -87,7 +91,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
|
||||
path = url + path
|
||||
|
||||
if not steps and path not in orig_url:
|
||||
versions.append((path, pv))
|
||||
versions.append((path, pv, HANDLER_NAME, CONFIDENCE))
|
||||
|
||||
if steps:
|
||||
ret = scan_directory_recursive(cp, ver, rev, path, steps, orig_url)
|
||||
@ -99,7 +103,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
|
||||
def scan(cpv, url):
|
||||
for bu in SCANDIR_BLACKLIST_URLS:
|
||||
if re.match(bu, url):
|
||||
euscan.output.einfo("%s is blacklisted by rule %s" % (url, bu))
|
||||
output.einfo("%s is blacklisted by rule %s" % (url, bu))
|
||||
return []
|
||||
|
||||
resolved_url = helpers.parse_mirror(url)
|
||||
@ -112,23 +116,25 @@ def scan(cpv, url):
|
||||
if ver not in resolved_url:
|
||||
newver = helpers.version_change_end_sep(ver)
|
||||
if newver and newver in resolved_url:
|
||||
euscan.output.einfo(
|
||||
output.einfo(
|
||||
"Version: using %s instead of %s" % (newver, ver)
|
||||
)
|
||||
ver = newver
|
||||
|
||||
template = helpers.template_from_url(resolved_url, ver)
|
||||
if '${' not in template:
|
||||
euscan.output.einfo(
|
||||
output.einfo(
|
||||
"Url doesn't seems to depend on version: %s not found in %s" %
|
||||
(ver, resolved_url)
|
||||
)
|
||||
return []
|
||||
else:
|
||||
euscan.output.einfo("Scanning: %s" % template)
|
||||
output.einfo("Scanning: %s" % template)
|
||||
|
||||
steps = helpers.generate_scan_paths(template)
|
||||
return scan_directory_recursive(cp, ver, rev, "", steps, url)
|
||||
ret = scan_directory_recursive(cp, ver, rev, "", steps, url)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def brute_force(cpv, url):
|
||||
@ -140,37 +146,37 @@ def brute_force(cpv, url):
|
||||
|
||||
for bp in BRUTEFORCE_BLACKLIST_PACKAGES:
|
||||
if re.match(bp, cp):
|
||||
euscan.output.einfo("%s is blacklisted by rule %s" % (cp, bp))
|
||||
output.einfo("%s is blacklisted by rule %s" % (cp, bp))
|
||||
return []
|
||||
|
||||
for bp in BRUTEFORCE_BLACKLIST_URLS:
|
||||
if re.match(bp, url):
|
||||
euscan.output.einfo("%s is blacklisted by rule %s" % (cp, bp))
|
||||
output.einfo("%s is blacklisted by rule %s" % (cp, bp))
|
||||
return []
|
||||
|
||||
euscan.output.einfo("Generating version from " + ver)
|
||||
output.einfo("Generating version from " + ver)
|
||||
|
||||
components = helpers.split_version(ver)
|
||||
versions = helpers.gen_versions(components, CONFIG["brute-force"])
|
||||
|
||||
""" Remove unwanted versions """
|
||||
# Remove unwanted versions
|
||||
for v in versions:
|
||||
if helpers.vercmp(cp, ver, helpers.join_version(v)) >= 0:
|
||||
versions.remove(v)
|
||||
|
||||
if not versions:
|
||||
euscan.output.einfo("Can't generate new versions from " + ver)
|
||||
output.einfo("Can't generate new versions from " + ver)
|
||||
return []
|
||||
|
||||
template = helpers.template_from_url(url, ver)
|
||||
|
||||
if '${PV}' not in template:
|
||||
euscan.output.einfo(
|
||||
output.einfo(
|
||||
"Url doesn't seems to depend on full version: %s not found in %s" %
|
||||
(ver, url))
|
||||
return []
|
||||
else:
|
||||
euscan.output.einfo("Brute forcing: %s" % template)
|
||||
output.einfo("Brute forcing: %s" % template)
|
||||
|
||||
result = []
|
||||
|
||||
@ -195,10 +201,11 @@ def brute_force(cpv, url):
|
||||
if not infos:
|
||||
continue
|
||||
|
||||
result.append([url, version])
|
||||
result.append([url, version, BRUTEFORCE_HANDLER_NAME,
|
||||
BRUTEFORCE_CONFIDENCE])
|
||||
|
||||
if len(result) > CONFIG['brute-force-false-watermark']:
|
||||
euscan.output.einfo(
|
||||
output.einfo(
|
||||
"Broken server detected ! Skipping brute force."
|
||||
)
|
||||
return []
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
from euscan.handlers import generic
|
||||
|
||||
HANDLER_NAME = "kde"
|
||||
|
||||
|
||||
def can_handle(cpv, url):
|
||||
if url.startswith('mirror://kde/'):
|
||||
@ -10,10 +12,10 @@ def can_handle(cpv, url):
|
||||
def clean_results(results):
|
||||
ret = []
|
||||
|
||||
for path, version in results:
|
||||
for path, version, confidence in results:
|
||||
if version == '5SUMS':
|
||||
continue
|
||||
ret.append((path, version))
|
||||
ret.append((path, version, HANDLER_NAME, confidence))
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
@ -3,8 +3,10 @@ import portage
|
||||
import urllib2
|
||||
import xml.dom.minidom
|
||||
|
||||
from euscan import helpers
|
||||
import euscan
|
||||
from euscan import helpers, output
|
||||
|
||||
HANDLER_NAME = "php"
|
||||
CONFIDENCE = 100.0
|
||||
|
||||
|
||||
def can_handle(cpv, url):
|
||||
@ -34,7 +36,7 @@ def scan(cpv, url):
|
||||
orig_url = url
|
||||
url = 'http://%s/rest/r/%s/allreleases.xml' % (channel, pkg.lower())
|
||||
|
||||
euscan.output.einfo("Using: " + url)
|
||||
output.einfo("Using: " + url)
|
||||
|
||||
try:
|
||||
fp = helpers.urlopen(url)
|
||||
@ -64,7 +66,7 @@ def scan(cpv, url):
|
||||
if url == orig_url:
|
||||
continue
|
||||
|
||||
ret.append((url, pv))
|
||||
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
@ -3,8 +3,10 @@ import re
|
||||
|
||||
import portage
|
||||
|
||||
from euscan import helpers
|
||||
import euscan
|
||||
from euscan import helpers, output
|
||||
|
||||
HANDLER_NAME = "pypi"
|
||||
CONFIDENCE = 100.0
|
||||
|
||||
|
||||
def can_handle(cpv, url):
|
||||
@ -26,7 +28,7 @@ def scan(cpv, url):
|
||||
|
||||
package = guess_package(cpv, url)
|
||||
|
||||
euscan.output.einfo("Using PyPi XMLRPC: " + package)
|
||||
output.einfo("Using PyPi XMLRPC: " + package)
|
||||
|
||||
client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
|
||||
versions = client.package_releases(package)
|
||||
@ -46,7 +48,7 @@ def scan(cpv, url):
|
||||
continue
|
||||
urls = client.release_urls(package, up_pv)
|
||||
urls = " ".join([infos['url'] for infos in urls])
|
||||
ret.append((urls, pv))
|
||||
ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
@ -3,8 +3,10 @@ import portage
|
||||
import json
|
||||
import urllib2
|
||||
|
||||
from euscan import helpers
|
||||
import euscan
|
||||
from euscan import helpers, output
|
||||
|
||||
HANDLER_NAME = "rubygem"
|
||||
CONFIDENCE = 100.0
|
||||
|
||||
|
||||
def can_handle(cpv, url):
|
||||
@ -31,13 +33,13 @@ def scan(cpv, url):
|
||||
|
||||
gem = guess_gem(cpv, url)
|
||||
if not gem:
|
||||
euscan.output.eerror("Can't guess gem name using %s and %s" % \
|
||||
output.eerror("Can't guess gem name using %s and %s" % \
|
||||
(cpv, url))
|
||||
return []
|
||||
|
||||
url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
|
||||
|
||||
euscan.output.einfo("Using: " + url)
|
||||
output.einfo("Using: " + url)
|
||||
|
||||
try:
|
||||
fp = helpers.urlopen(url)
|
||||
@ -65,7 +67,7 @@ def scan(cpv, url):
|
||||
if helpers.version_filtered(cp, ver, pv):
|
||||
continue
|
||||
url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
|
||||
ret.append((url, pv))
|
||||
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
@ -37,6 +37,18 @@ _v_end = '((-|_)(pre|p|beta|b|alpha|a|rc|r)\d*)'
|
||||
_v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))'
|
||||
|
||||
|
||||
def get_version_type(version):
|
||||
types = []
|
||||
gentoo_types = ("alpha", "beta", "pre", "rc", "p")
|
||||
|
||||
for token in re.findall("[\._-]([a-zA-Z]+)", version):
|
||||
if token in gentoo_types:
|
||||
types.append(token)
|
||||
if types:
|
||||
return types[0]
|
||||
return "release"
|
||||
|
||||
|
||||
# Stolen from g-pypi
|
||||
def gentoo_mangle_version(up_pv):
|
||||
"""Convert PV to MY_PV if needed
|
||||
@ -121,7 +133,7 @@ def gentoo_mangle_version(up_pv):
|
||||
pv = up_pv = rev_match.group(1)
|
||||
replace_me = rev_match.group(2)
|
||||
rev = rev_match.group(3)
|
||||
additional_version = '.' + rev
|
||||
additional_version = '_p' + rev
|
||||
|
||||
for this_suf in suf_matches.keys():
|
||||
if rs_match:
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import portage
|
||||
from portage.dbapi import porttree
|
||||
@ -10,8 +10,7 @@ from gentoolkit.package import Package
|
||||
from gentoolkit.eclean.search import (port_settings)
|
||||
|
||||
from euscan import CONFIG, BLACKLIST_PACKAGES
|
||||
from euscan import handlers
|
||||
from euscan import helpers
|
||||
from euscan import handlers, helpers, output
|
||||
from euscan.ebuild import package_from_ebuild
|
||||
|
||||
import euscan
|
||||
@ -19,19 +18,27 @@ import euscan
|
||||
def filter_versions(cp, versions):
|
||||
filtered = {}
|
||||
|
||||
for url, version in versions:
|
||||
for url, version, handler, confidence in versions:
|
||||
|
||||
''' Try to keep the most specific urls (determinted by the length) '''
|
||||
# Try to keep the most specific urls (determinted by the length)
|
||||
if version in filtered and len(url) < len(filtered[version]):
|
||||
continue
|
||||
|
||||
''' Remove blacklisted versions '''
|
||||
# Remove blacklisted versions
|
||||
if helpers.version_blacklisted(cp, version):
|
||||
continue
|
||||
|
||||
filtered[version] = url
|
||||
filtered[version] = {
|
||||
"url": url,
|
||||
"handler": handler,
|
||||
"confidence": confidence
|
||||
}
|
||||
|
||||
return [(cp, filtered[version], version) for version in filtered]
|
||||
return [
|
||||
(cp, filtered[version]["url"], version, filtered[version]["handler"],
|
||||
filtered[version]["confidence"])
|
||||
for version in filtered
|
||||
]
|
||||
|
||||
|
||||
def scan_upstream_urls(cpv, urls):
|
||||
@ -39,22 +46,22 @@ def scan_upstream_urls(cpv, urls):
|
||||
|
||||
for filename in urls:
|
||||
for url in urls[filename]:
|
||||
if not CONFIG['quiet']:
|
||||
if not CONFIG['quiet'] and not CONFIG['format']:
|
||||
pp.uprint()
|
||||
euscan.output.einfo("SRC_URI is '%s'" % url)
|
||||
output.einfo("SRC_URI is '%s'" % url)
|
||||
|
||||
if '://' not in url:
|
||||
euscan.output.einfo("Invalid url '%s'" % url)
|
||||
output.einfo("Invalid url '%s'" % url)
|
||||
continue
|
||||
|
||||
''' Try normal scan '''
|
||||
# Try normal scan
|
||||
if CONFIG["scan-dir"]:
|
||||
versions.extend(handlers.scan(cpv, url))
|
||||
|
||||
if versions and CONFIG['oneshot']:
|
||||
break
|
||||
|
||||
''' Brute Force '''
|
||||
# Brute Force
|
||||
if CONFIG["brute-force"] > 0:
|
||||
versions.extend(handlers.brute_force(cpv, url))
|
||||
|
||||
@ -91,10 +98,10 @@ def scan_upstream(query):
|
||||
)
|
||||
|
||||
if not matches:
|
||||
sys.stderr.write(
|
||||
output.ewarn(
|
||||
pp.warn("No package matching '%s'" % pp.pkgquery(query))
|
||||
)
|
||||
return []
|
||||
return None
|
||||
|
||||
matches = sorted(matches)
|
||||
pkg = matches.pop()
|
||||
@ -103,29 +110,42 @@ def scan_upstream(query):
|
||||
pkg = matches.pop()
|
||||
|
||||
if not pkg:
|
||||
sys.stderr.write(pp.warn("Package '%s' only have a dev version (9999)"
|
||||
% pp.pkgquery(pkg.cp)))
|
||||
return []
|
||||
output.ewarn(
|
||||
pp.warn("Package '%s' only have a dev version (9999)"
|
||||
% pp.pkgquery(pkg.cp))
|
||||
)
|
||||
return None
|
||||
|
||||
# useful data only for formatted output
|
||||
start_time = datetime.now()
|
||||
output.metadata("datetime", start_time.isoformat(), show=False)
|
||||
output.metadata("cp", pkg.cp, show=False)
|
||||
output.metadata("cpv", pkg.cpv, show=False)
|
||||
|
||||
if pkg.cp in BLACKLIST_PACKAGES:
|
||||
sys.stderr.write(
|
||||
output.ewarn(
|
||||
pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp))
|
||||
)
|
||||
return []
|
||||
return None
|
||||
|
||||
if not CONFIG['quiet']:
|
||||
if not CONFIG['format']:
|
||||
pp.uprint(
|
||||
" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name()))
|
||||
)
|
||||
pp.uprint()
|
||||
else:
|
||||
output.metadata("overlay", pp.section(pkg.repo_name()))
|
||||
|
||||
ebuild_path = pkg.ebuild_path()
|
||||
if ebuild_path:
|
||||
pp.uprint('Ebuild: ' + pp.path(os.path.normpath(ebuild_path)))
|
||||
output.metadata(
|
||||
"ebuild", pp.path(os.path.normpath(ebuild_path))
|
||||
)
|
||||
|
||||
pp.uprint('Repository: ' + pkg.repo_name())
|
||||
pp.uprint('Homepage: ' + pkg.environment("HOMEPAGE"))
|
||||
pp.uprint('Description: ' + pkg.environment("DESCRIPTION"))
|
||||
output.metadata("repository", pkg.repo_name())
|
||||
output.metadata("homepage", pkg.environment("HOMEPAGE"))
|
||||
output.metadata("description", pkg.environment("DESCRIPTION"))
|
||||
|
||||
cpv = pkg.cpv
|
||||
metadata = {
|
||||
@ -137,15 +157,19 @@ def scan_upstream(query):
|
||||
alist = porttree._parse_uri_map(cpv, metadata, use=use)
|
||||
aalist = porttree._parse_uri_map(cpv, metadata)
|
||||
except Exception as e:
|
||||
sys.stderr.write(pp.warn("%s\n" % str(e)))
|
||||
sys.stderr.write(
|
||||
output.ewarn(pp.warn("%s\n" % str(e)))
|
||||
output.ewarn(
|
||||
pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv))
|
||||
)
|
||||
return []
|
||||
return None
|
||||
|
||||
if "mirror" in portage.settings.features:
|
||||
urls = aalist
|
||||
else:
|
||||
urls = alist
|
||||
|
||||
# output scan time for formatted output
|
||||
scan_time = (datetime.now() - start_time).total_seconds()
|
||||
output.metadata("scan_time", scan_time, show=False)
|
||||
|
||||
return scan_upstream_urls(pkg.cpv, urls)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user