src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Run black on project

Browse Source 
Signed-off-by: Alfred Wingate <parona@protonmail.com>
This commit is contained in:
Alfred Wingate 2023-11-14 23:03:53 +02:00
parent a91775919c
commit d860708ec9
No known key found for this signature in database
GPG Key ID: A12750536B5E7010
26 changed files with 616 additions and 565 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

278
bin/euscan
View File

@ -35,8 +35,8 @@ from euscan._version import __version__
# Globals
isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty()
isatty_stderr = os.environ.get('TERM') != 'dumb' and sys.stderr.isatty()
isatty = os.environ.get("TERM") != "dumb" and sys.stdout.isatty()
isatty_stderr = os.environ.get("TERM") != "dumb" and sys.stderr.isatty()
def exit_helper(status):
@ -62,8 +62,7 @@ def setup_signals():
def print_version():
"""Output the version info."""
print("%s (%s) - %s" \
% (__productname__, __version__, __description__))
print("%s (%s) - %s" % (__productname__, __version__, __description__))
print()
print("Author: %s <%s>" % (__author__, __email__))
print("Copyright 2011 Gentoo Foundation")
@ -78,78 +77,134 @@ def print_usage(_error=None, help=None):
if _error:
out = sys.stderr
if not _error in ('global-options', 'packages',):
if not _error in (
"global-options",
"packages",
):
_error = None
if not _error and not help:
help = 'all'
help = "all"
if _error in ('global-options',):
if _error in ("global-options",):
output.eerror("Wrong option on command line.\n")
if _error in ('packages',):
if _error in ("packages",):
output.eerror("You need to specify exactly one package.\n")
print(white("Usage:"), file=out)
if _error in ('global-options', 'packages',) or help == 'all':
print(" " + turquoise(__productname__),
if (
_error
in (
"global-options",
"packages",
)
or help == "all"
):
print(
" " + turquoise(__productname__),
yellow("[options]"),
green("<package> [<package> [...]]"), file=out)
if _error in ('global-options',) or help == 'all':
print(" " + turquoise(__productname__),
yellow("[--help, --version]"), file=out)
green("<package> [<package> [...]]"),
file=out,
)
if _error in ("global-options",) or help == "all":
print(" " + turquoise(__productname__), yellow("[--help, --version]"), file=out)
print(file=out)
if _error in ('global-options',) or help:
if _error in ("global-options",) or help:
print("Available ", yellow("options") + ":", file=out)
print(yellow(" -C, --nocolor") +
" - turn off colors on output", file=out)
print(yellow(" -q, --quiet") +
" - be as quiet as possible", file=out)
print(yellow(" -h, --help") +
" - display the help screen", file=out)
print(yellow(" -V, --version") +
" - display version info", file=out)
print(
yellow(" -C, --nocolor")
+ " - turn off colors on output",
file=out,
)
print(
yellow(" -q, --quiet")
+ " - be as quiet as possible",
file=out,
)
print(
yellow(" -h, --help")
+ " - display the help screen",
file=out,
)
print(
yellow(" -V, --version") + " - display version info",
file=out,
)
print(file=out)
print(yellow(" -1, --oneshot") +
" - stop as soon as a new version is found",
file=out)
print(yellow(" -b, --brute-force=<level>") +
" - define the brute force " + yellow("<level>") +
" (default: 2)\n" +
" " * 38 + "bigger levels will generate more versions numbers\n" +
" " * 38 + "0 means disabled", file=out)
print(yellow(" -f, --format=<format>") +
" - define the output " + yellow("<format>") +
" (available: json, xml)", file=out)
print(yellow(" -p, --progress") +
" - display a progress bar", file=out)
print(yellow(" -i, --ignore-pre-release") +
" " * 11 + "- Ignore non-stable versions", file=out)
print(yellow(" -I, --ignore-pre-release-if-stable") +
" - Ignore non-stable versions only if current\n" +
" " * 38 + "version is stable", file=out)
print(yellow(" --mirror") +
" - use mirror:// URIs", file=out)
print(yellow(" --ebuild-uri") +
" - use ebuild variables in URIs", file=out)
print(yellow(" --no-handlers") +
" - exclude handlers (comma-separated list)",
file=out)
print(
yellow(" -1, --oneshot")
+ " - stop as soon as a new version is found",
file=out,
)
print(
yellow(" -b, --brute-force=<level>")
+ " - define the brute force "
+ yellow("<level>")
+ " (default: 2)\n"
+ " " * 38
+ "bigger levels will generate more versions numbers\n"
+ " " * 38
+ "0 means disabled",
file=out,
)
print(
yellow(" -f, --format=<format>")
+ " - define the output "
+ yellow("<format>")
+ " (available: json, xml)",
file=out,
)
print(
yellow(" -p, --progress") + " - display a progress bar",
file=out,
)
print(
yellow(" -i, --ignore-pre-release")
+ " " * 11
+ "- Ignore non-stable versions",
file=out,
)
print(
yellow(" -I, --ignore-pre-release-if-stable")
+ " - Ignore non-stable versions only if current\n"
+ " " * 38
+ "version is stable",
file=out,
)
print(
yellow(" --mirror") + " - use mirror:// URIs",
file=out,
)
print(
yellow(" --ebuild-uri")
+ " - use ebuild variables in URIs",
file=out,
)
print(
yellow(" --no-handlers")
+ " - exclude handlers (comma-separated list)",
file=out,
)
print(file=out)
if _error in ('packages',) or help:
print(green(" package") +
" " * 28 + "- the packages (or ebuilds) you want to scan",
file=out)
if _error in ("packages",) or help:
print(
green(" package")
+ " " * 28
+ "- the packages (or ebuilds) you want to scan",
file=out,
)
print(file=out)
#print( "More detailed instruction can be found in",
#turquoise("`man %s`" % __productname__), file=out)
# print( "More detailed instruction can be found in",
# turquoise("`man %s`" % __productname__), file=out)
class ParseArgsException(Exception):
"""For parseArgs() -> main() communications."""
def __init__(self, value):
self.value = value
@ -168,92 +223,102 @@ def parse_args():
return_code = True
for o, a in opts:
if o in ("-h", "--help"):
raise ParseArgsException('help')
raise ParseArgsException("help")
elif o in ("-V", "--version"):
raise ParseArgsException('version')
raise ParseArgsException("version")
elif o in ("-C", "--nocolor"):
CONFIG['nocolor'] = True
CONFIG["nocolor"] = True
pp.output.nocolor()
elif o in ("-q", "--quiet"):
CONFIG['quiet'] = True
CONFIG['verbose'] = 0
CONFIG["quiet"] = True
CONFIG["verbose"] = 0
elif o in ("-1", "--oneshot"):
CONFIG['oneshot'] = True
CONFIG["oneshot"] = True
elif o in ("-b", "--brute-force"):
CONFIG['brute-force'] = int(a)
elif o in ("-v", "--verbose") and not CONFIG['quiet']:
CONFIG['verbose'] += 1
CONFIG["brute-force"] = int(a)
elif o in ("-v", "--verbose") and not CONFIG["quiet"]:
CONFIG["verbose"] += 1
elif o in ("-f", "--format"):
CONFIG['format'] = a
CONFIG['nocolor'] = True
CONFIG["format"] = a
CONFIG["nocolor"] = True
pp.output.nocolor()
elif o in ("-p", "--progress"):
CONFIG['progress'] = isatty_stderr
CONFIG["progress"] = isatty_stderr
elif o in ("--mirror"):
CONFIG['mirror'] = True
CONFIG["mirror"] = True
elif o in ("-i", "--ignore-pre-release"):
CONFIG['ignore-pre-release'] = True
CONFIG["ignore-pre-release"] = True
elif o in ("-I", "--ignore-pre-release-if-stable"):
CONFIG['ignore-pre-release-if-stable'] = True
CONFIG["ignore-pre-release-if-stable"] = True
elif o in ("--ebuild-uri"):
CONFIG['ebuild-uri'] = True
CONFIG["ebuild-uri"] = True
elif o in ("--no-handlers"):
CONFIG['handlers-exclude'] = a.split(",")
CONFIG["handlers-exclude"] = a.split(",")
else:
return_code = False
return return_code
# here are the different allowed command line options (getopt args)
getopt_options = {'short': {}, 'long': {}}
getopt_options['short']['global'] = "hVCqv1b:f:piI"
getopt_options['long']['global'] = [
"help", "version", "nocolor", "quiet", "verbose", "oneshot",
"brute-force=", "format=", "progress", "mirror", "ignore-pre-release",
"ignore-pre-release-if-stable", "ebuild-uri", "no-handlers="
getopt_options = {"short": {}, "long": {}}
getopt_options["short"]["global"] = "hVCqv1b:f:piI"
getopt_options["long"]["global"] = [
"help",
"version",
"nocolor",
"quiet",
"verbose",
"oneshot",
"brute-force=",
"format=",
"progress",
"mirror",
"ignore-pre-release",
"ignore-pre-release-if-stable",
"ebuild-uri",
"no-handlers=",
]
short_opts = getopt_options['short']['global']
long_opts = getopt_options['long']['global']
opts_mode = 'global'
short_opts = getopt_options["short"]["global"]
long_opts = getopt_options["long"]["global"]
opts_mode = "global"
# apply getopts to command line, show partial help on failure
try:
opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts)
except:
raise ParseArgsException(opts_mode + '-options')
raise ParseArgsException(opts_mode + "-options")
# set options accordingly
option_switch(opts)
if len(args) < 1:
raise ParseArgsException('packages')
raise ParseArgsException("packages")
return args
def main():
"""Parse command line and execute all actions."""
CONFIG['nocolor'] = (
CONFIG['nocolor'] or
(settings["NOCOLOR"] in ('yes', 'true') or not isatty)
CONFIG["nocolor"] = CONFIG["nocolor"] or (
settings["NOCOLOR"] in ("yes", "true") or not isatty
)
if CONFIG['nocolor']:
if CONFIG["nocolor"]:
pp.output.nocolor()
# parse command line options and actions
try:
queries = parse_args()
except ParseArgsException as e:
if e.value == 'help':
print_usage(help='all')
if e.value == "help":
print_usage(help="all")
exit_helper(0)
elif e.value[:5] == 'help-':
elif e.value[:5] == "help-":
print_usage(help=e.value[5:])
exit_helper(0)
elif e.value == 'version':
elif e.value == "version":
print_version()
exit_helper(0)
@ -261,14 +326,14 @@ def main():
print_usage(e.value)
exit_helper(EINVAL)
if CONFIG['verbose'] > 2:
if CONFIG["verbose"] > 2:
HTTPConnection.debuglevel = 1
if not CONFIG["format"] and not CONFIG['quiet']:
if not CONFIG["format"] and not CONFIG["quiet"]:
CONFIG["progress"] = False
on_progress = None
if CONFIG['progress']:
if CONFIG["progress"]:
on_progress_gen = progress_bar()
on_progress = next(on_progress_gen)
on_progress(maxval=len(queries) * 100, increment=0, label="Working...")
@ -293,35 +358,36 @@ def main():
from os.path import basename # To get the short name
output.eerror(
"The short ebuild name '%s' is ambiguous. Please specify" %
basename(pkgs[0]) +
"one of the above fully-qualified ebuild names instead."
"The short ebuild name '%s' is ambiguous. Please specify"
% basename(pkgs[0])
+ "one of the above fully-qualified ebuild names instead."
)
exit_helper(1)
except GentoolkitException as err:
output.eerror('%s: %s' % (query, str(err)))
output.eerror("%s: %s" % (query, str(err)))
exit_helper(1)
except Exception as err:
import traceback
print ('-' * 60)
traceback.print_exc(file=sys.stderr)
print ('-' * 60)
output.eerror('%s: %s' % (query, str(err)))
print("-" * 60)
traceback.print_exc(file=sys.stderr)
print("-" * 60)
output.eerror("%s: %s" % (query, str(err)))
exit_helper(1)
if not ret and not CONFIG['quiet']:
if not ret and not CONFIG["quiet"]:
output.einfo(
"Didn't find any new version, check package's homepage " +
"for more informations"
"Didn't find any new version, check package's homepage "
+ "for more informations"
)
if not (CONFIG['format'] or CONFIG['quiet']) and len(queries) > 1:
if not (CONFIG["format"] or CONFIG["quiet"]) and len(queries) > 1:
print("")
if CONFIG['progress']:
if CONFIG["progress"]:
next(on_progress_gen)
print("\n", file=sys.stderr)

70
bin/euscan_patch_metadata
View File

@ -28,25 +28,25 @@ def guess_indent_values(before):
def guess_for_tags(tags):
for tag in tags:
for i in [0, 2, 4, 6, 8, 12, 16]:
if '\n%s<%s' % (' ' * i, tag) in before:
if "\n%s<%s" % (" " * i, tag) in before:
return i, False
for i in [0, 1, 2]:
if '\n%s<%s' % ('\t' * i, tag) in before:
if "\n%s<%s" % ("\t" * i, tag) in before:
return i, True
return -1, False
rindent, tab = guess_for_tags(
['herd', 'maintainer', 'longdescription', 'use', 'upstream']
["herd", "maintainer", "longdescription", "use", "upstream"]
)
if rindent == -1:
rindent = 2
rindent_str = ('\t' if tab else ' ') * rindent
indent, tab = guess_for_tags(['watch', 'name', 'email'])
rindent_str = ("\t" if tab else " ") * rindent
indent, tab = guess_for_tags(["watch", "name", "email"])
if indent == -1:
indent = rindent * 2 if rindent else 4
if rindent and rindent_str == '\t':
if rindent and rindent_str == "\t":
tab = True
indent_str = ('\t' if tab else ' ') * indent
indent_str = ("\t" if tab else " ") * indent
return rindent_str, indent_str
@ -66,7 +66,7 @@ def handle_diff(deb_url):
watch_data = ""
fp = gzip.open(temp_deb, 'rb')
fp = gzip.open(temp_deb, "rb")
for line in fp:
if re.match(r"\+\+\+ .+?/debian/watch", line):
fp.readline() # diff lines, don't care
@ -144,8 +144,9 @@ def patch_metadata(package, watch_data, diff=False):
data = original
# clean watch_data
watch_data = "\n".join([line for line in watch_data.split("\n")
if not line.startswith("#")]) # comments
watch_data = "\n".join(
[line for line in watch_data.split("\n") if not line.startswith("#")]
) # comments
watch_data = watch_data.replace("\\\n", "") # remove backslashes
@ -163,10 +164,7 @@ def patch_metadata(package, watch_data, diff=False):
continue
# parse watch_line
result = re.match(
r'(?:opts=(?:"([^"]+?)"|([^\s]+?)) )?(.*)',
watch_line
)
result = re.match(r'(?:opts=(?:"([^"]+?)"|([^\s]+?)) )?(.*)', watch_line)
opts_quote, opts, url = result.groups()
opts = opts_quote or opts
@ -188,21 +186,27 @@ def patch_metadata(package, watch_data, diff=False):
url = " ".join([x for x in url_search.groups() if x is not None])
if opts:
watch_tag = '%s<watch version="%s" %s>%s</watch>' % \
(indent, version, opts, url)
watch_tag = '%s<watch version="%s" %s>%s</watch>' % (
indent,
version,
opts,
url,
)
else:
watch_tag = '%s<watch version="%s">%s</watch>' % \
(indent, version, url)
watch_tag = '%s<watch version="%s">%s</watch>' % (indent, version, url)
watch_tags.append(watch_tag)
watch_tags = "\n".join(watch_tags)
if '<upstream>' in data:
data = data.replace('<upstream>', '<upstream>\n%s' % watch_tags, 1)
if "<upstream>" in data:
data = data.replace("<upstream>", "<upstream>\n%s" % watch_tags, 1)
else:
rep = '%s<upstream>\n%s\n%s</upstream>\n</pkgmetadata>' % \
(rindent, watch_tags, rindent)
data = data.replace('</pkgmetadata>', rep, 1)
rep = "%s<upstream>\n%s\n%s</upstream>\n</pkgmetadata>" % (
rindent,
watch_tags,
rindent,
)
data = data.replace("</pkgmetadata>", rep, 1)
if not diff:
return data
@ -214,8 +218,8 @@ def patch_metadata(package, watch_data, diff=False):
res = unified_diff(
original.splitlines(True),
data.splitlines(True),
fromfile=os.path.join('a/', metadata_path),
tofile=os.path.join('b/', metadata_path),
fromfile=os.path.join("a/", metadata_path),
tofile=os.path.join("b/", metadata_path),
)
return "".join([x for x in res])
@ -240,7 +244,7 @@ def process_package(query, diff=False):
matches = sorted(matches)
package = matches.pop()
if '9999' in package.version and len(matches) > 0:
if "9999" in package.version and len(matches) > 0:
package = matches.pop()
watch_data = get_watch_data(package)
@ -252,16 +256,21 @@ def process_package(query, diff=False):
def main():
import optparse
p = optparse.OptionParser(
usage="usage: %prog <package> [<package> [...]]",
)
p.add_option('-d', '--diff', action="store_true", dest="diff",
p.add_option(
"-d",
"--diff",
action="store_true",
dest="diff",
default=False,
help="Outputs a diff")
help="Outputs a diff",
)
opts, packages = p.parse_args()
logging.basicConfig(stream=sys.stderr, level=logging.INFO,
format='%(message)s')
logging.basicConfig(stream=sys.stderr, level=logging.INFO, format="%(message)s")
for package in packages:
logger.info("Processing %s..." % package)
@ -269,5 +278,6 @@ def main():
if result:
sys.stdout.write(result)
if __name__ == "__main__":
main()

85
src/euscan/__init__.py
View File

@ -9,30 +9,30 @@ from ast import literal_eval
CONFIG = {
'nocolor': False,
'quiet': False,
'verbose': 1,
'debug': False,
'brute-force': 3,
'brute-force-recursive': True,
'brute-force-false-watermark': 50,
'scan-dir': True,
'oneshot': True,
'user-agent': 'escan (http://euscan.iksaif.net)',
'skip-robots-txt': False,
'cache': False,
'format': None,
'indent': 2,
'progress': False,
'mirror': False,
'ignore-pre-release': False,
'ignore-pre-release-if-stable': False,
'ebuild-uri': False,
'handlers-exclude': [],
"nocolor": False,
"quiet": False,
"verbose": 1,
"debug": False,
"brute-force": 3,
"brute-force-recursive": True,
"brute-force-false-watermark": 50,
"scan-dir": True,
"oneshot": True,
"user-agent": "escan (http://euscan.iksaif.net)",
"skip-robots-txt": False,
"cache": False,
"format": None,
"indent": 2,
"progress": False,
"mirror": False,
"ignore-pre-release": False,
"ignore-pre-release-if-stable": False,
"ebuild-uri": False,
"handlers-exclude": [],
}
config = configparser.ConfigParser()
config.read(['/etc/euscan.conf', os.path.expanduser('~/.euscan.conf')])
config.read(["/etc/euscan.conf", os.path.expanduser("~/.euscan.conf")])
if config.has_section("euscan"):
for key, value in config.items("euscan"):
if key in CONFIG:
@ -41,49 +41,50 @@ if config.has_section("euscan"):
BLACKLIST_VERSIONS = [
# Compatibility package for running binaries linked against a
# pre gcc 3.4 libstdc++, won't be updated
'>=sys-libs/libstdc++-v3-3.4',
">=sys-libs/libstdc++-v3-3.4",
# Actually older or incorrect
'~app-backup/backup-manager-0.7.15',
'=x11-plugins/wmacpimon-001',
"~app-backup/backup-manager-0.7.15",
"=x11-plugins/wmacpimon-001",
]
BLACKLIST_PACKAGES = [
# These kernels are almost dead
'sys-kernel/xbox-sources',
"sys-kernel/xbox-sources",
]
SCANDIR_BLACKLIST_URLS = [
'mirror://rubygems/(.*)', # Not browsable
'mirror://gentoo/(.*)' # Directory too big
"mirror://rubygems/(.*)", # Not browsable
"mirror://gentoo/(.*)", # Directory too big
]
BRUTEFORCE_BLACKLIST_PACKAGES = [
# infinite loop any
# http://plone.org/products/plonepopoll/releases/*/plonepopoll-2-6-1.tgz
# link will work
'net-zope/plonepopoll'
"net-zope/plonepopoll"
]
BRUTEFORCE_BLACKLIST_URLS = [
'http://hydra.nixos.org/build/(.*)', # infinite loop
"http://hydra.nixos.org/build/(.*)", # infinite loop
# Doesn't respect 404, infinite loop
'http://www.rennings.net/gentoo/distfiles/(.*)',
'http://art.gnome.org/download/(.*)',
'http://barelysufficient.org/~olemarkus/(.*)',
'http://olemarkus.org/~olemarkus/(.*)',
"http://www.rennings.net/gentoo/distfiles/(.*)",
"http://art.gnome.org/download/(.*)",
"http://barelysufficient.org/~olemarkus/(.*)",
"http://olemarkus.org/~olemarkus/(.*)",
]
ROBOTS_TXT_BLACKLIST_DOMAINS = [
'(.*)sourceforge(.*)',
'(.*)github.com',
'(.*)qt\.nokia\.com(.*)',
'(.*)chromium\.org(.*)',
'(.*)nodejs\.org(.*)',
'(.*)download\.mono-project\.com(.*)',
'(.*)fedorahosted\.org(.*)',
'(.*)download\.tuxfamily\.org(.*)',
'(.*)festvox\.org(.*)',
"(.*)sourceforge(.*)",
"(.*)github.com",
"(.*)qt\.nokia\.com(.*)",
"(.*)chromium\.org(.*)",
"(.*)nodejs\.org(.*)",
"(.*)download\.mono-project\.com(.*)",
"(.*)fedorahosted\.org(.*)",
"(.*)download\.tuxfamily\.org(.*)",
"(.*)festvox\.org(.*)",
]
from euscan.out import EuscanOutput
output = EuscanOutput(CONFIG)

37
src/euscan/ebuild.py
View File

@ -24,12 +24,10 @@ def package_from_ebuild(ebuild):
# since the canonical path returned from os.getcwd() may may be
# unusable in cases where the directory stucture is built from
# symlinks.
pwd = os.environ.get('PWD', '')
pwd = os.environ.get("PWD", "")
if sys.hexversion < 0x3000000:
pwd = _unicode_decode(pwd, encoding=_encodings['content'],
errors='strict')
if pwd and pwd != mycwd and \
os.path.realpath(pwd) == mycwd:
pwd = _unicode_decode(pwd, encoding=_encodings["content"], errors="strict")
if pwd and pwd != mycwd and os.path.realpath(pwd) == mycwd:
mycwd = portage.normalize_path(pwd)
ebuild = os.path.join(mycwd, ebuild)
@ -38,22 +36,29 @@ def package_from_ebuild(ebuild):
# subdirectories of the base can be built from symlinks (like crossdev
# does).
ebuild_portdir = os.path.realpath(
os.path.dirname(os.path.dirname(os.path.dirname(ebuild))))
os.path.dirname(os.path.dirname(os.path.dirname(ebuild)))
)
ebuild = os.path.join(ebuild_portdir, *ebuild.split(os.path.sep)[-3:])
vdb_path = os.path.join(portage.settings['ROOT'], VDB_PATH)
vdb_path = os.path.join(portage.settings["ROOT"], VDB_PATH)
# Make sure that portdb.findname() returns the correct ebuild.
if ebuild_portdir != vdb_path and \
ebuild_portdir not in portage.portdb.porttrees:
if ebuild_portdir != vdb_path and ebuild_portdir not in portage.portdb.porttrees:
if sys.hexversion >= 0x3000000:
os.environ["PORTDIR_OVERLAY"] = \
os.environ.get("PORTDIR_OVERLAY", "") + \
" " + _shell_quote(ebuild_portdir)
os.environ["PORTDIR_OVERLAY"] = (
os.environ.get("PORTDIR_OVERLAY", "")
+ " "
+ _shell_quote(ebuild_portdir)
)
else:
os.environ["PORTDIR_OVERLAY"] = \
os.environ.get("PORTDIR_OVERLAY", "") + \
" " + _unicode_encode(_shell_quote(ebuild_portdir),
encoding=_encodings['content'], errors='strict')
os.environ["PORTDIR_OVERLAY"] = (
os.environ.get("PORTDIR_OVERLAY", "")
+ " "
+ _unicode_encode(
_shell_quote(ebuild_portdir),
encoding=_encodings["content"],
errors="strict",
)
)
portage.close_portdbapi_caches()
importlib.reload(portage)

85
src/euscan/handlers/__init__.py
View File

@ -6,31 +6,27 @@ from euscan import CONFIG, output
from portage.xml.metadata import MetaDataXML
handlers = {'package': [], 'url': [], 'all': {}}
handlers = {"package": [], "url": [], "all": {}}
# autoimport all modules in this directory and append them to handlers list
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
module = loader.find_spec(module_name).loader.load_module(module_name)
if not hasattr(module, 'HANDLER_NAME'):
if not hasattr(module, "HANDLER_NAME"):
continue
if hasattr(module, 'scan_url'):
handlers['url'].append(module)
if hasattr(module, 'scan_pkg'):
handlers['package'].append(module)
handlers['all'][module.HANDLER_NAME] = module
if hasattr(module, "scan_url"):
handlers["url"].append(module)
if hasattr(module, "scan_pkg"):
handlers["package"].append(module)
handlers["all"][module.HANDLER_NAME] = module
# sort handlers by priority
def sort_handlers(handlers):
return sorted(
handlers,
key=lambda handler: handler.PRIORITY,
reverse=True
)
return sorted(handlers, key=lambda handler: handler.PRIORITY, reverse=True)
handlers['package'] = sort_handlers(handlers['package'])
handlers['url'] = sort_handlers(handlers['url'])
handlers["package"] = sort_handlers(handlers["package"])
handlers["url"] = sort_handlers(handlers["url"])
def find_best_handler(kind, pkg, *args):
@ -38,8 +34,9 @@ def find_best_handler(kind, pkg, *args):
Find the best handler for the given package
"""
for handler in handlers[kind]:
if (handler.HANDLER_NAME not in CONFIG["handlers-exclude"] and
handler.can_handle(pkg, *args)):
if handler.HANDLER_NAME not in CONFIG[
"handlers-exclude"
] and handler.can_handle(pkg, *args):
return handler
return None
@ -49,8 +46,8 @@ def find_handlers(kind, names):
for name in names:
# Does this handler exist, and handle this kind of thing ? (pkg / url)
if name in handlers['all'] and handlers['all'][name] in handlers[kind]:
ret.append(handlers['all'][name])
if name in handlers["all"] and handlers["all"][name] in handlers[kind]:
ret.append(handlers["all"][name])
return ret
@ -60,17 +57,16 @@ def get_metadata(pkg):
pkg_metadata = None
meta_override = os.path.join('metadata', pkg.category, pkg.name,
'metadata.xml')
meta_override = os.path.join("metadata", pkg.category, pkg.name, "metadata.xml")
try:
if os.path.exists(meta_override):
pkg_metadata = MetaDataXML(meta_override)
output.einfo('Using custom metadata: %s' % meta_override)
output.einfo("Using custom metadata: %s" % meta_override)
if not pkg_metadata:
pkg_metadata = pkg.metadata
except Exception as e:
output.ewarn('Error when fetching metadata: %s' % str(e))
output.ewarn("Error when fetching metadata: %s" % str(e))
if not pkg_metadata:
return {}
@ -79,13 +75,13 @@ def get_metadata(pkg):
for upstream in pkg_metadata._xml_tree.findall("upstream"):
for node in upstream.findall("watch"):
options = dict(node.attrib)
options['data'] = node.text
options["data"] = node.text
if "type" in options:
handler = options['type']
handler = options["type"]
else:
handler = "url"
options['type'] = "url"
options["type"] = "url"
for key in ["versionmangle", "downloadurlmangle"]:
value = options.get(key, None)
@ -103,10 +99,10 @@ def get_metadata(pkg):
continue
if handler in metadata:
for i in range(len(metadata[handler])):
if not metadata[handler][i]['data']:
metadata[handler][i]['data'] = node.text
if not metadata[handler][i]["data"]:
metadata[handler][i]["data"] = node.text
else:
metadata[handler] = [{'type': handler, 'data': node.text}]
metadata[handler] = [{"type": handler, "data": node.text}]
return metadata
@ -145,24 +141,21 @@ def scan_url(pkg, urls, options, on_progress=None):
output.einfo("SRC_URI is '%s'" % url)
if '://' not in url:
if "://" not in url:
output.einfo("Invalid url '%s'" % url)
continue
try:
url_handler = find_best_handler('url', pkg, url)
url_handler = find_best_handler("url", pkg, url)
if url_handler:
for o in options:
versions += url_handler.scan_url(pkg, url, o)
else:
output.eerror("Can't find a suitable handler!")
except Exception as e:
output.ewarn(
"Handler failed: [%s] %s" %
(e.__class__.__name__, str(e))
)
output.ewarn("Handler failed: [%s] %s" % (e.__class__.__name__, str(e)))
if versions and CONFIG['oneshot']:
if versions and CONFIG["oneshot"]:
break
if on_progress and progress_available > 0:
@ -178,15 +171,15 @@ def scan(pkg, urls, on_progress=None):
in url handling.
"""
if not CONFIG['quiet'] and not CONFIG['format']:
sys.stdout.write('\n')
if not CONFIG["quiet"] and not CONFIG["format"]:
sys.stdout.write("\n")
metadata = get_metadata(pkg)
versions = []
pkg_handlers = find_handlers('package', list(metadata.keys()))
pkg_handlers = find_handlers("package", list(metadata.keys()))
if not pkg_handlers:
pkg_handler = find_best_handler('package', pkg)
pkg_handler = find_best_handler("package", pkg)
if pkg_handler:
pkg_handlers = [pkg_handler]
@ -201,17 +194,17 @@ def scan(pkg, urls, on_progress=None):
def mangle(kind, name, string):
if name not in handlers['all']:
if name not in handlers["all"]:
return None
handler = handlers['all'][name]
if not hasattr(handler, 'mangle_%s' % kind):
handler = handlers["all"][name]
if not hasattr(handler, "mangle_%s" % kind):
return None
return getattr(handler, 'mangle_%s' % kind)(string)
return getattr(handler, "mangle_%s" % kind)(string)
def mangle_url(name, string):
return mangle('url', name, string)
return mangle("url", name, string)
def mangle_version(name, string):
return mangle('version', name, string)
return mangle("version", name, string)

12
src/euscan/handlers/berlios.py
View File

@ -36,19 +36,13 @@ def scan_url(pkg, url, options):
project_page = "http://developer.berlios.de/projects/%s" % project
content = urllib.request.urlopen(project_page).read()
project_id = re.search(
r"/project/filelist.php\?group_id=(\d+)",
content
).group(1)
project_id = re.search(r"/project/filelist.php\?group_id=(\d+)", content).group(1)
base_url = (
"http://developer.berlios.de/project/filelist.php?group_id=%s" %
project_id
"http://developer.berlios.de/project/filelist.php?group_id=%s" % project_id
)
file_pattern = regex_from_template(
filename.replace(ver, "${PV}")
)
file_pattern = regex_from_template(filename.replace(ver, "${PV}"))
result = url_scan(pkg, base_url, file_pattern)

44
src/euscan/handlers/cpan.py
View File

@ -13,7 +13,7 @@ _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*")
def can_handle(pkg, url=None):
return url and url.startswith('mirror://cpan/')
return url and url.startswith("mirror://cpan/")
def guess_package(cp, url):
@ -24,7 +24,7 @@ def guess_package(cp, url):
if match:
pkg = match.group(1)
try:
cp, ver, rev = portage.pkgsplit('fake/' + pkg)
cp, ver, rev = portage.pkgsplit("fake/" + pkg)
except:
pass
@ -34,7 +34,7 @@ def guess_package(cp, url):
def mangle_version(up_pv):
if up_pv.startswith('v'):
if up_pv.startswith("v"):
return up_pv[1:]
# clean
@ -53,7 +53,7 @@ def mangle_version(up_pv):
if len(splitted) == 2: # Split second part is sub-groups
part = splitted.pop()
for i in range(0, len(part), 3):
splitted.append(part[i:i + 3])
splitted.append(part[i : i + 3])
if len(splitted) == 2: # add last group if it's missing
splitted.append("0")
@ -78,11 +78,11 @@ def mangle_version(up_pv):
def cpan_mangle_version(pv):
pos = pv.find('.')
pos = pv.find(".")
if pos <= 0:
return pv
up_pv = pv.replace('.', '')
up_pv = up_pv[0:pos] + '.' + up_pv[pos:]
up_pv = pv.replace(".", "")
up_pv = up_pv[0:pos] + "." + up_pv[pos:]
return up_pv
@ -99,17 +99,17 @@ def scan_url(pkg, url, options):
output.einfo("Using CPAN API: %s", remote_pkg)
return scan_pkg(pkg, {'data': remote_pkg})
return scan_pkg(pkg, {"data": remote_pkg})
def scan_pkg(pkg, options):
remote_pkg = options['data']
remote_pkg = options["data"]
# Defaults to CPAN mangling rules
if 'versionmangle' not in options:
options['versionmangle'] = ['cpan', 'gentoo']
if "versionmangle" not in options:
options["versionmangle"] = ["cpan", "gentoo"]
url = 'http://search.cpan.org/api/dist/%s' % remote_pkg
url = "http://search.cpan.org/api/dist/%s" % remote_pkg
cp, ver, rev = pkg.cp, pkg.version, pkg.revision
m_ver = cpan_mangle_version(ver)
@ -128,19 +128,19 @@ def scan_pkg(pkg, options):
data = fp.read()
data = json.loads(data)
if 'releases' not in data:
if "releases" not in data:
return []
ret = []
for version in data['releases']:
#if version['status'] == 'testing':
for version in data["releases"]:
# if version['status'] == 'testing':
# continue
up_pv = version['version']
up_pv = version["version"]
pv = mangling.mangle_version(up_pv, options)
if up_pv.startswith('v'):
if up_pv.startswith("v"):
if helpers.version_filtered(cp, ver, pv):
continue
else:
@ -148,11 +148,11 @@ def scan_pkg(pkg, options):
if helpers.version_filtered(cp, m_ver, m_pv, cpan_vercmp):
continue
url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % (
version['cpanid'][0],
version['cpanid'][0:1],
version['cpanid'],
version['archive']
url = "mirror://cpan/authors/id/%s/%s/%s/%s" % (
version["cpanid"][0],
version["cpanid"][0:1],
version["cpanid"],
version["archive"],
)
url = mangling.mangle_url(url, options)

2
src/euscan/handlers/deb.py
View File

@ -19,7 +19,7 @@ def can_handle(pkg, url=None):
def scan_pkg(pkg, options):
cp, ver, rev = portage.pkgsplit(pkg.cpv)
packages_url, package_name = options['data'].strip().split(" ", 1)
packages_url, package_name = options["data"].strip().split(" ", 1)
output.einfo("Using Debian Packages: " + packages_url)

13
src/euscan/handlers/freecode.py
View File

@ -17,7 +17,7 @@ def can_handle(pkg, url=None):
def scan_pkg(pkg, options):
cp, ver, rev = portage.pkgsplit(pkg.cpv)
package = options['data'].strip()
package = options["data"].strip()
output.einfo("Using FreeCode handler: " + package)
@ -25,8 +25,7 @@ def scan_pkg(pkg, options):
content = str(fp.read())
result = re.findall(
r'<a href="/projects/%s/releases/(\d+)">([^<]+)</a>' % package,
content
r'<a href="/projects/%s/releases/(\d+)">([^<]+)</a>' % package, content
)
ret = []
@ -34,15 +33,15 @@ def scan_pkg(pkg, options):
pv = mangling.mangle_version(up_pv, options)
if helpers.version_filtered(cp, ver, pv):
continue
fp = urllib.request.urlopen("http://freecode.com/projects/%s/releases/%s" %
(package, release_id))
fp = urllib.request.urlopen(
"http://freecode.com/projects/%s/releases/%s" % (package, release_id)
)
content = str(fp.read())
download_page = re.findall(r'<a href="(/urls/[^"]+)"', content)[0]
fp = urllib.request.urlopen("http://freecode.com%s" % download_page)
content = str(fp.read())
url = re.findall(
r'In case it doesn\'t, click here: <a href="([^"]+)"',
content
r'In case it doesn\'t, click here: <a href="([^"]+)"', content
)[0]
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret

54
src/euscan/handlers/generic.py
View File

@ -11,8 +11,15 @@ except ImportError:
import portage
from euscan import output, helpers, mangling, CONFIG, SCANDIR_BLACKLIST_URLS, \
BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS
from euscan import (
output,
helpers,
mangling,
CONFIG,
SCANDIR_BLACKLIST_URLS,
BRUTEFORCE_BLACKLIST_PACKAGES,
BRUTEFORCE_BLACKLIST_URLS,
)
HANDLER_NAME = "generic"
CONFIDENCE = 45
@ -55,7 +62,7 @@ def scan_html(data, url, pattern):
soup = BeautifulSoup(data, features="lxml")
results = []
for link in soup.findAll('a'):
for link in soup.findAll("a"):
href = link.get("href")
if not href:
continue
@ -66,11 +73,9 @@ def scan_html(data, url, pattern):
match = re.search(pattern, href, re.I)
if match:
results.append(
(".".join([x for x in match.groups() if x is not None]),
match.group(0))
(".".join([x for x in match.groups() if x is not None]), match.group(0))
)
return results
@ -83,8 +88,7 @@ def scan_ftp(data, url, pattern):
match = re.search(pattern, line, re.I)
if match:
results.append(
(".".join([x for x in match.groups() if x is not None]),
match.group(0))
(".".join([x for x in match.groups() if x is not None]), match.group(0))
)
return results
@ -116,7 +120,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url, options):
if re.search(b"<\s*a\s+[^>]*href", data, re.I):
results.extend(scan_html(data, url, pattern))
elif url.startswith('ftp://'):
elif url.startswith("ftp://"):
results.extend(scan_ftp(data, url, pattern))
versions = []
@ -136,8 +140,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url, options):
versions.append((path, pv, HANDLER_NAME, confidence))
if steps:
ret = scan_directory_recursive(cp, ver, rev, path, steps, orig_url,
options)
ret = scan_directory_recursive(cp, ver, rev, path, steps, orig_url, options)
versions.extend(ret)
return versions
@ -160,16 +163,14 @@ def scan_url(pkg, url, options):
if ver not in resolved_url:
newver = helpers.version_change_end_sep(ver)
if newver and newver in resolved_url:
output.einfo(
"Version: using %s instead of %s" % (newver, ver)
)
output.einfo("Version: using %s instead of %s" % (newver, ver))
ver = newver
template = helpers.template_from_url(resolved_url, ver)
if '${' not in template:
if "${" not in template:
output.einfo(
"Url doesn't seems to depend on version: %s not found in %s" %
(ver, resolved_url)
"Url doesn't seems to depend on version: %s not found in %s"
% (ver, resolved_url)
)
return []
else:
@ -220,10 +221,11 @@ def brute_force(pkg, url):
template = helpers.template_from_url(url, ver)
if '${PV}' not in template:
if "${PV}" not in template:
output.einfo(
"Url doesn't seems to depend on full version: %s not found in %s" %
(ver, url))
"Url doesn't seems to depend on full version: %s not found in %s"
% (ver, url)
)
return []
else:
output.einfo("Brute forcing: %s" % template)
@ -250,19 +252,15 @@ def brute_force(pkg, url):
if not infos:
continue
confidence = confidence_score(try_url, url,
minimum=BRUTEFORCE_CONFIDENCE)
confidence = confidence_score(try_url, url, minimum=BRUTEFORCE_CONFIDENCE)
result.append([try_url, version, BRUTEFORCE_HANDLER_NAME, confidence])
if len(result) > CONFIG['brute-force-false-watermark']:
output.einfo(
"Broken server detected ! Skipping brute force."
)
if len(result) > CONFIG["brute-force-false-watermark"]:
output.einfo("Broken server detected ! Skipping brute force.")
return []
if CONFIG["brute-force-recursive"]:
for v in helpers.gen_versions(list(components),
CONFIG["brute-force"]):
for v in helpers.gen_versions(list(components), CONFIG["brute-force"]):
if v not in versions and tuple(v) not in done:
versions.append(v)

26
src/euscan/handlers/github.py
View File

@ -12,18 +12,18 @@ PRIORITY = 90
def can_handle(pkg, url=None):
return url and url.startswith('mirror://github/')
return url and url.startswith("mirror://github/")
def guess_package(cp, url):
match = re.search('^mirror://github/(.*?)/(.*?)/(.*)$', url)
match = re.search("^mirror://github/(.*?)/(.*?)/(.*)$", url)
assert(match)
assert match
return (match.group(1), match.group(2), match.group(3))
def scan_url(pkg, url, options):
'http://developer.github.com/v3/repos/downloads/'
"http://developer.github.com/v3/repos/downloads/"
user, project, filename = guess_package(pkg.cpv, url)
@ -35,25 +35,27 @@ def scan_url(pkg, url, options):
# now create a filename-matching regexp
# XXX: supposedly replace first with (?P<foo>...)
# and remaining ones with (?P=foo)
fnre = re.compile('^%s$' % \
re.escape(filename).replace(re.escape(ver), '(.*?)'))
fnre = re.compile("^%s$" % re.escape(filename).replace(re.escape(ver), "(.*?)"))
output.einfo("Using github API for: project=%s user=%s filename=%s" % \
(project, user, filename))
output.einfo(
"Using github API for: project=%s user=%s filename=%s"
% (project, user, filename)
)
dlreq = urllib.request.urlopen('https://api.github.com/repos/%s/%s/downloads' % \
(user, project))
dlreq = urllib.request.urlopen(
"https://api.github.com/repos/%s/%s/downloads" % (user, project)
)
dls = json.load(dlreq)
ret = []
for dl in dls:
m = fnre.match(dl['name'])
m = fnre.match(dl["name"])
if m:
pv = mangling.mangle_version(m.group(1), options)
if helpers.version_filtered(cp, ver, pv):
continue
url = mangling.mangle_url(dl['html_url'], options)
url = mangling.mangle_url(dl["html_url"], options)
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret

27
src/euscan/handlers/gnome.py
View File

@ -16,15 +16,15 @@ HANDLER_NAME = "gnome"
CONFIDENCE = 100
PRIORITY = 90
GNOME_URL_SOURCE = 'http://ftp.gnome.org/pub/GNOME/sources'
GNOME_URL_SOURCE = "http://ftp.gnome.org/pub/GNOME/sources"
def can_handle(_pkg, url=None):
return url and url.startswith('mirror://gnome/')
return url and url.startswith("mirror://gnome/")
def guess_package(cp, url):
match = re.search('mirror://gnome/sources/([^/]+)/.*', url)
match = re.search("mirror://gnome/sources/([^/]+)/.*", url)
if match:
return match.group(1)
@ -34,27 +34,27 @@ def guess_package(cp, url):
def scan_url(pkg, url, options):
'http://ftp.gnome.org/pub/GNOME/sources/'
"http://ftp.gnome.org/pub/GNOME/sources/"
package = {
'data': guess_package(pkg.cpv, url),
'type': 'gnome',
"data": guess_package(pkg.cpv, url),
"type": "gnome",
}
return scan_pkg(pkg, package)
def scan_pkg(pkg, options):
package = options['data']
package = options["data"]
output.einfo("Using Gnome json cache: " + package)
fp = urllib.request.urlopen('/'.join([GNOME_URL_SOURCE, package, 'cache.json']))
fp = urllib.request.urlopen("/".join([GNOME_URL_SOURCE, package, "cache.json"]))
content = fp.read()
fp.close()
cache = json.loads(content, encoding='ascii')
cache = json.loads(content, encoding="ascii")
if cache[0] != 4:
output.eerror('Unknow cache format detected')
output.eerror("Unknow cache format detected")
return []
versions = cache[2][package]
@ -72,13 +72,12 @@ def scan_pkg(pkg, options):
if helpers.version_filtered(cp, ver, pv):
continue
up_files = cache[1][package][up_pv]
for tarball_comp in ('tar.xz', 'tar.bz2', 'tar.gz'):
for tarball_comp in ("tar.xz", "tar.bz2", "tar.gz"):
if tarball_comp in up_files:
url = '/'.join([GNOME_URL_SOURCE, package,
up_files[tarball_comp]])
url = "/".join([GNOME_URL_SOURCE, package, up_files[tarball_comp]])
break
else:
output.ewarn('No tarball for release %s' % up_pv)
output.ewarn("No tarball for release %s" % up_pv)
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret

5
src/euscan/handlers/google_code.py
View File

@ -23,6 +23,7 @@ def can_handle(pkg, url=None):
return re.match(package_name_regex, url)
def scan_url(pkg, url, options):
output.einfo("Using Google Code handler")
@ -31,9 +32,7 @@ def scan_url(pkg, url, options):
package_name = re.match(package_name_regex, url).group(1)
base_url = "http://code.google.com/p/%s/downloads/list" % package_name
file_pattern = regex_from_template(
url.split("/")[-1].replace(ver, "${PV}")
)
file_pattern = regex_from_template(url.split("/")[-1].replace(ver, "${PV}"))
result = url_scan(pkg, base_url, file_pattern)

13
src/euscan/handlers/kde.py
View File

@ -6,14 +6,14 @@ HANDLER_NAME = "kde"
def can_handle(pkg, url):
return url and url.startswith('mirror://kde/')
return url and url.startswith("mirror://kde/")
def clean_results(results):
ret = []
for path, version, _, confidence in results:
if version == '5SUMS':
if version == "5SUMS":
continue
ret.append((path, version, HANDLER_NAME, confidence))
@ -23,16 +23,15 @@ def clean_results(results):
def scan_url(pkg, url, options):
results = generic.scan(pkg.cpv, url)
if generic.startswith('mirror://kde/unstable/'):
url = generic.replace('mirror://kde/unstable/', 'mirror://kde/stable/')
if generic.startswith("mirror://kde/unstable/"):
url = generic.replace("mirror://kde/unstable/", "mirror://kde/stable/")
results += generic.scan(pkg.cpv, url)
if not results: # if nothing was found go brute forcing
results = generic.brute_force(pkg.cpv, url)
if generic.startswith('mirror://kde/unstable/'):
url = generic.replace('mirror://kde/unstable/',
'mirror://kde/stable/')
if generic.startswith("mirror://kde/unstable/"):
url = generic.replace("mirror://kde/unstable/", "mirror://kde/stable/")
results += generic.brute_force(pkg.cpv, url)
return clean_results(results)

3
src/euscan/handlers/pear.py
View File

@ -6,7 +6,8 @@ PRIORITY = 90
def can_handle(pkg, url=None):
return url and url.startswith('http://%s.php.net/get/' % HANDLER_NAME)
return url and url.startswith("http://%s.php.net/get/" % HANDLER_NAME)
scan_url = php.scan_url
scan_pkg = php.scan_pkg

4
src/euscan/handlers/pecl.py
View File

@ -4,8 +4,10 @@ HANDLER_NAME = "pecl"
CONFIDENCE = 100
PRIORITY = 90
def can_handle(pkg, url=None):
return url and url.startswith('http://%s.php.net/get/' % HANDLER_NAME)
return url and url.startswith("http://%s.php.net/get/" % HANDLER_NAME)
scan_url = php.scan_url
scan_pkg = php.scan_pkg

15
src/euscan/handlers/php.py
View File

@ -9,11 +9,13 @@ HANDLER_NAME = "php"
CONFIDENCE = 100
PRIORITY = 90
def can_handle(pkg, url=None):
return False
def guess_package_and_channel(cp, url):
match = re.search('http://(.*)\.php\.net/get/(.*)-(.*).tgz', url)
match = re.search("http://(.*)\.php\.net/get/(.*)-(.*).tgz", url)
if match:
host = match.group(1)
@ -26,15 +28,16 @@ def guess_package_and_channel(cp, url):
def scan_url(pkg, url, options):
package, channel = guess_package_and_channel(pkg.cp, url)
return scan_pkg(pkg, {'type' : channel, 'data' : package })
return scan_pkg(pkg, {"type": channel, "data": package})
def scan_pkg(pkg, options):
cp, ver, rev = pkg.cp, pkg.version, pkg.revision
package = options['data']
channel = options['type']
package = options["data"]
channel = options["type"]
url = 'http://%s.php.net/rest/r/%s/allreleases.xml' % (channel, package.lower())
url = "http://%s.php.net/rest/r/%s/allreleases.xml" % (channel, package.lower())
output.einfo("Using: " + url)
@ -61,7 +64,7 @@ def scan_pkg(pkg, options):
if helpers.version_filtered(cp, ver, pv):
continue
url = 'http://%s.php.net/get/%s-%s.tgz' % (channel, package, up_pv)
url = "http://%s.php.net/get/%s-%s.tgz" % (channel, package, up_pv)
url = mangling.mangle_url(url, options)
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))

15
src/euscan/handlers/pypi.py
View File

@ -11,11 +11,11 @@ PRIORITY = 90
def can_handle(pkg, url=None):
return url and url.startswith('mirror://pypi/')
return url and url.startswith("mirror://pypi/")
def guess_package(cp, url):
match = re.search('mirror://pypi/\w+/(.*)/.*', url)
match = re.search("mirror://pypi/\w+/(.*)/.*", url)
if match:
return match.group(1)
@ -25,18 +25,18 @@ def guess_package(cp, url):
def scan_url(pkg, url, options):
'http://wiki.python.org/moin/PyPiXmlRpc'
"http://wiki.python.org/moin/PyPiXmlRpc"
package = guess_package(pkg.cpv, url)
return scan_pkg(pkg, {'data': package})
return scan_pkg(pkg, {"data": package})
def scan_pkg(pkg, options):
package = options['data']
package = options["data"]
output.einfo("Using PyPi XMLRPC: " + package)
client = xmlrpc.client.ServerProxy('https://pypi.python.org/pypi')
client = xmlrpc.client.ServerProxy("https://pypi.python.org/pypi")
versions = client.package_releases(package)
if not versions:
@ -52,7 +52,6 @@ def scan_pkg(pkg, options):
if helpers.version_filtered(cp, ver, pv):
continue
urls = client.release_urls(package, up_pv)
urls = " ".join([mangling.mangle_url(infos['url'], options)
for infos in urls])
urls = " ".join([mangling.mangle_url(infos["url"], options) for infos in urls])
ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
return ret

21
src/euscan/handlers/rubygems.py
View File

@ -11,13 +11,13 @@ PRIORITY = 90
def can_handle(pkg, url=None):
return url and url.startswith('mirror://rubygems/')
return url and url.startswith("mirror://rubygems/")
def guess_gem(cpv, url):
match = re.search('mirror://rubygems/(.*).gem', url)
match = re.search("mirror://rubygems/(.*).gem", url)
if match:
cpv = 'fake/%s' % match.group(1)
cpv = "fake/%s" % match.group(1)
ret = portage.pkgsplit(cpv)
if not ret:
@ -30,23 +30,22 @@ def guess_gem(cpv, url):
def scan_url(pkg, url, options):
'http://guides.rubygems.org/rubygems-org-api/#gemversion'
"http://guides.rubygems.org/rubygems-org-api/#gemversion"
gem = guess_gem(pkg.cpv, url)
if not gem:
output.eerror("Can't guess gem name using %s and %s" % \
(pkg.cpv, url))
output.eerror("Can't guess gem name using %s and %s" % (pkg.cpv, url))
return []
output.einfo("Using RubyGem API: %s" % gem)
return scan_pkg(pkg, {'data': gem})
return scan_pkg(pkg, {"data": gem})
def scan_pkg(pkg, options):
gem = options['data']
url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
gem = options["data"]
url = "http://rubygems.org/api/v1/versions/%s.json" % gem
try:
fp = helpers.urlopen(url)
@ -65,11 +64,11 @@ def scan_pkg(pkg, options):
ret = []
for version in versions:
up_pv = version['number']
up_pv = version["number"]
pv = mangling.mangle_version(up_pv, options)
if helpers.version_filtered(cp, ver, pv):
continue
url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
url = "http://rubygems.org/gems/%s-%s.gem" % (gem, up_pv)
url = mangling.mangle_url(url, options)
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret

7
src/euscan/handlers/sourceforge.py
View File

@ -28,14 +28,11 @@ def scan_url(pkg, url, options):
cp, ver, rev = portage.pkgsplit(pkg.cpv)
project, filename = re.search(
"mirror://sourceforge/([^/]+)/(?:.*/)?([^/]+)",
url
"mirror://sourceforge/([^/]+)/(?:.*/)?([^/]+)", url
).groups()
base_url = "http://qa.debian.org/watch/sf.php/%s" % project
file_pattern = regex_from_template(
filename.replace(ver, "${PV}")
)
file_pattern = regex_from_template(filename.replace(ver, "${PV}"))
result = url_scan(pkg, base_url, file_pattern)

13
src/euscan/handlers/url.py
View File

@ -33,7 +33,7 @@ def handle_directory_patterns(base, file_pattern):
i += 1
basedir = "/".join(basedir)
directory_pattern = splitted[i]
final = "/".join(splitted[i + 1:])
final = "/".join(splitted[i + 1 :])
try:
fp = helpers.urlopen(basedir)
@ -52,15 +52,14 @@ def handle_directory_patterns(base, file_pattern):
else:
scan_data = generic.scan_html(data, basedir, directory_pattern)
return [("/".join((basedir, path, final)), file_pattern)
for _, path in scan_data]
return [("/".join((basedir, path, final)), file_pattern) for _, path in scan_data]
def read_options(options):
try:
base, file_pattern = options['data'].split(" ")[:2]
base, file_pattern = options["data"].split(" ")[:2]
except ValueError:
base, file_pattern = options['data'], None
base, file_pattern = options["data"], None
# the file pattern can be in the base url
pattern_regex = r"/([^/]*\([^/]*\)[^/]*)$"
@ -70,9 +69,7 @@ def read_options(options):
base = base.replace(file_pattern, "")
# handle sf.net specially
base = base.replace(
"http://sf.net/", "http://qa.debian.org/watch/sf.php/"
)
base = base.replace("http://sf.net/", "http://qa.debian.org/watch/sf.php/")
return base, file_pattern

126
src/euscan/helpers.py
View File

@ -21,18 +21,17 @@ from euscan.version import parse_version
def htop_vercmp(a, b):
def fixver(v):
if v in ['0.11', '0.12', '0.13']:
v = '0.1.' + v[3:]
if v in ["0.11", "0.12", "0.13"]:
v = "0.1." + v[3:]
return v
return simple_vercmp(fixver(a), fixver(b))
VERSION_CMP_PACKAGE_QUIRKS = {
'sys-process/htop': htop_vercmp
}
_v_end = r'(?:(?:-|_)(?:pre|p|beta|b|alpha|a|rc|r)\d*)'
_v = r'((?:\d+)(?:(?:\.\d+)*)(?:[a-zA-Z]*?)(?:' + _v_end + '*))'
VERSION_CMP_PACKAGE_QUIRKS = {"sys-process/htop": htop_vercmp}
_v_end = r"(?:(?:-|_)(?:pre|p|beta|b|alpha|a|rc|r)\d*)"
_v = r"((?:\d+)(?:(?:\.\d+)*)(?:[a-zA-Z]*?)(?:" + _v_end + "*))"
def cast_int_components(version):
@ -75,15 +74,15 @@ def version_is_nightly(a, b):
b = parse_version(b)
# Try to skip nightly builds when not wanted (www-apps/moodle)
if len(a) != len(b) and len(b) == 2 and len(b[0]) == len('yyyymmdd'):
if b[0][:4] != '0000':
if len(a) != len(b) and len(b) == 2 and len(b[0]) == len("yyyymmdd"):
if b[0][:4] != "0000":
return True
return False
def version_blacklisted(cp, version):
rule = None
cpv = '%s-%s' % (cp, version)
cpv = "%s-%s" % (cp, version)
# Check that the generated cpv can be used by portage
if not portage.versions.catpkgsplit(cpv):
@ -104,10 +103,10 @@ def version_change_end_sep(version):
if not match:
return None
end = match.group(1)
if end[0] == '_':
newend = end.replace('_', '-')
elif end[0] == '-':
newend = end.replace('-', '_')
if end[0] == "_":
newend = end.replace("_", "-")
elif end[0] == "-":
newend = end.replace("-", "_")
else:
return None
return version.replace(end, newend)
@ -135,17 +134,17 @@ def generate_templates_vars(version):
var = []
for j in range(i):
ver.append(str(part[j]))
var.append('${%d}' % j)
var.append("${%d}" % j)
ret.append((".".join(ver), ".".join(var)))
ret.append((version, '${PV}'))
ret.append((version, "${PV}"))
ret.reverse()
return ret
def template_from_url(url, version):
prefix, chunks = url.split('://')
chunks = chunks.split('/')
prefix, chunks = url.split("://")
chunks = chunks.split("/")
for i in range(len(chunks)):
chunk = chunks[i]
@ -162,9 +161,9 @@ def template_from_url(url, version):
def url_from_template(url, version):
components = split_version(version)
url = url.replace('${PV}', version)
url = url.replace("${PV}", version)
for i in range(len(components)):
url = url.replace('${%d}' % i, str(components[i]))
url = url.replace("${%d}" % i, str(components[i]))
return url
@ -172,8 +171,8 @@ def url_from_template(url, version):
# Stolen from distutils.LooseVersion
# Used for brute force to increment the version
def split_version(version):
component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE)
components = [x for x in component_re.split(version) if x and x != '.']
component_re = re.compile(r"(\d+ | [a-z]+ | \.)", re.VERBOSE)
components = [x for x in component_re.split(version) if x and x != "."]
for i in range(len(components)):
try:
components[i] = int(components[i])
@ -229,7 +228,7 @@ def gen_versions(components, level):
def timeout_for_url(url):
if 'sourceforge' in url:
if "sourceforge" in url:
timeout = 15
else:
timeout = 5
@ -246,7 +245,7 @@ rpcache = {}
def urlallowed(url):
if CONFIG['skip-robots-txt']:
if CONFIG["skip-robots-txt"]:
return True
protocol, domain = urllib.parse.urlparse(url)[:2]
@ -255,15 +254,15 @@ def urlallowed(url):
if re.match(bd, domain):
return True
for d in ['sourceforge', 'berlios', 'github.com']:
for d in ["sourceforge", "berlios", "github.com"]:
if d in domain:
return True
if protocol == 'ftp':
if protocol == "ftp":
return True
baseurl = '%s://%s' % (protocol, domain)
robotsurl = urllib.parse.urljoin(baseurl, 'robots.txt')
baseurl = "%s://%s" % (protocol, domain)
robotsurl = urllib.parse.urljoin(baseurl, "robots.txt")
if baseurl in rpcache:
rp = rpcache[baseurl]
@ -283,7 +282,7 @@ def urlallowed(url):
setdefaulttimeout(timeout)
return rp.can_fetch(CONFIG['user-agent'], url) if rp else True
return rp.can_fetch(CONFIG["user-agent"], url) if rp else True
def urlopen(url, timeout=None, verb="GET"):
@ -294,23 +293,24 @@ def urlopen(url, timeout=None, verb="GET"):
if not timeout:
timeout = timeout_for_url(url)
if verb == 'GET':
if verb == "GET":
request = urllib.request.Request(url)
elif verb == 'HEAD':
elif verb == "HEAD":
request = HeadRequest(url)
else:
return None
request.add_header('User-Agent', CONFIG['user-agent'])
request.add_header("User-Agent", CONFIG["user-agent"])
handlers = []
if CONFIG['cache']:
if CONFIG["cache"]:
from cache import CacheHandler
handlers.append(CacheHandler(CONFIG['cache']))
if CONFIG['verbose']:
debuglevel = CONFIG['verbose'] - 1
handlers.append(CacheHandler(CONFIG["cache"]))
if CONFIG["verbose"]:
debuglevel = CONFIG["verbose"] - 1
handlers.append(urllib.request.HTTPHandler(debuglevel=debuglevel))
opener = urllib.request.build_opener(*handlers)
@ -330,7 +330,7 @@ def tryurl(fileurl, template):
try:
basename = os.path.basename(fileurl)
fp = urlopen(fileurl, verb='HEAD')
fp = urlopen(fileurl, verb="HEAD")
if not fp:
euscan.output.eend(errno.EPERM)
return None
@ -339,16 +339,17 @@ def tryurl(fileurl, template):
# Some URLs return Content-disposition with different filename
# Disable check for now (I have no seen false positives)
#if 'Content-disposition' in headers and \
# if 'Content-disposition' in headers and \
# basename not in headers['Content-disposition']:
# result = None
if 'Content-Length' in headers and headers['Content-Length'] == '0':
if "Content-Length" in headers and headers["Content-Length"] == "0":
result = None
elif 'Content-Type' in headers and \
'text/html' in headers['Content-Type']:
elif "Content-Type" in headers and "text/html" in headers["Content-Type"]:
result = None
elif 'Content-Type' in headers and \
'application/x-httpd-php' in headers['Content-Type']:
elif (
"Content-Type" in headers
and "application/x-httpd-php" in headers["Content-Type"]
):
result = None
elif fp.geturl() != fileurl:
regex = regex_from_template(template)
@ -356,8 +357,9 @@ def tryurl(fileurl, template):
basename2 = os.path.basename(fp.geturl())
# Redirect to another (earlier?) version
if basename != basename2 and (re.match(regex, fp.geturl()) or \
re.match(baseregex, basename2)):
if basename != basename2 and (
re.match(regex, fp.geturl()) or re.match(baseregex, basename2)
):
result = None
if result:
@ -378,33 +380,33 @@ def regex_from_template(template):
regexp = re.escape(template)
# Unescape specific stuff
regexp = regexp.replace('\$\{', '${')
regexp = regexp.replace('\}', '}')
regexp = regexp.replace('}\.$', '}.$')
regexp = regexp.replace("\$\{", "${")
regexp = regexp.replace("\}", "}")
regexp = regexp.replace("}\.$", "}.$")
# Replace ${\d+}
#regexp = regexp.replace('${0}', r'([\d]+?)')
regexp = re.sub(r'(\$\{\d+\}(\.?))+', r'([\\w\.]+?)', regexp)
# regexp = regexp.replace('${0}', r'([\d]+?)')
regexp = re.sub(r"(\$\{\d+\}(\.?))+", r"([\\w\.]+?)", regexp)
#regexp = re.sub(r'(\$\{\d+\}\.?)+', r'([\w]+?)', regexp)
#regexp = re.sub(r'(\$\{\d+\}\.+)+', '(.+?)\.', regexp)
#regexp = re.sub(r'(\$\{\d+\})+', '(.+?)', regexp)
# regexp = re.sub(r'(\$\{\d+\}\.?)+', r'([\w]+?)', regexp)
# regexp = re.sub(r'(\$\{\d+\}\.+)+', '(.+?)\.', regexp)
# regexp = re.sub(r'(\$\{\d+\})+', '(.+?)', regexp)
# Full version
regexp = regexp.replace('${PV}', _v)
regexp = regexp.replace("${PV}", _v)
# End
regexp = regexp + r'/?$'
regexp = regexp + r"/?$"
return regexp
def basedir_from_template(template):
idx = template.find('${')
idx = template.find("${")
if idx == -1:
return template
idx = template[0:idx].rfind('/')
idx = template[0:idx].rfind("/")
if idx == -1:
return ""
@ -412,15 +414,15 @@ def basedir_from_template(template):
def generate_scan_paths(url):
prefix, chunks = url.split('://')
chunks = chunks.split('/')
prefix, chunks = url.split("://")
chunks = chunks.split("/")
steps = []
path = prefix + ":/"
for chunk in chunks:
if '${' in chunk:
steps.append((path, '^(?:|.*/)' + regex_from_template(chunk)))
if "${" in chunk:
steps.append((path, "^(?:|.*/)" + regex_from_template(chunk)))
path = ""
else:
path += "/"
@ -444,7 +446,7 @@ def parse_mirror(uri):
return None
mirrorname = uri[9:eidx]
path = uri[eidx + 1:]
path = uri[eidx + 1 :]
if mirrorname in mirrors:
mirrors = mirrors[mirrorname]

58
src/euscan/mangling.py
View File

@ -31,11 +31,11 @@ def apply_mangling_rules(kind, rules, string):
ret = None
# First try handlers rules
if rule == 'gentoo' and kind == 'versionmangle':
if rule == "gentoo" and kind == "versionmangle":
ret = gentoo_mangle_version(string)
elif kind == 'downloadurlmangle':
elif kind == "downloadurlmangle":
ret = euscan.handlers.mangle_url(rule, string)
elif kind == 'versionmangle':
elif kind == "versionmangle":
ret = euscan.handlers.mangle_version(rule, string)
if ret is not None: # Use return value as new string if not None
@ -48,13 +48,13 @@ def apply_mangling_rules(kind, rules, string):
def mangle_version(up_pv, options):
# Default rule is gentoo when empty
if 'versionmangle' not in options or not options['versionmangle']:
options['versionmangle'] = ['gentoo']
return apply_mangling_rules('versionmangle', options, up_pv)
if "versionmangle" not in options or not options["versionmangle"]:
options["versionmangle"] = ["gentoo"]
return apply_mangling_rules("versionmangle", options, up_pv)
def mangle_url(url, options):
return apply_mangling_rules('downloadurlmangle', options, url)
return apply_mangling_rules("downloadurlmangle", options, url)
# Stolen from g-pypi
@ -107,29 +107,27 @@ def gentoo_mangle_version(up_pv):
number of match.groups every time to simplify the code
"""
bad_suffixes = re.compile(
r'((?:[._-]*)(?:dev|devel|final|stable|snapshot)$)', re.I)
revision_suffixes = re.compile(
r'(.*?)([\._-]*(?:r|patch|p)[\._-]*)([0-9]*)$', re.I)
bad_suffixes = re.compile(r"((?:[._-]*)(?:dev|devel|final|stable|snapshot)$)", re.I)
revision_suffixes = re.compile(r"(.*?)([\._-]*(?:r|patch|p)[\._-]*)([0-9]*)$", re.I)
suf_matches = {
'_pre': [
r'(.*?)([\._-]*dev[\._-]*r?)([0-9]+)$',
r'(.*?)([\._-]*(?:pre|preview)[\._-]*)([0-9]*)$',
"_pre": [
r"(.*?)([\._-]*dev[\._-]*r?)([0-9]+)$",
r"(.*?)([\._-]*(?:pre|preview)[\._-]*)([0-9]*)$",
],
'_alpha': [
r'(.*?)([\._-]*(?:alpha|test)[\._-]*)([0-9]*)$',
r'(.*?)([\._-]*a[\._-]*)([0-9]*)$',
r'(.*[^a-z])(a)([0-9]*)$',
"_alpha": [
r"(.*?)([\._-]*(?:alpha|test)[\._-]*)([0-9]*)$",
r"(.*?)([\._-]*a[\._-]*)([0-9]*)$",
r"(.*[^a-z])(a)([0-9]*)$",
],
'_beta': [
r'(.*?)([\._-]*beta[\._-]*)([0-9]*)$',
r'(.*?)([\._-]*b)([0-9]*)$',
r'(.*[^a-z])(b)([0-9]*)$',
"_beta": [
r"(.*?)([\._-]*beta[\._-]*)([0-9]*)$",
r"(.*?)([\._-]*b)([0-9]*)$",
r"(.*[^a-z])(b)([0-9]*)$",
],
'_rc': [
r'(.*?)([\._-]*rc[\._-]*)([0-9]*)$',
r'(.*?)([\._-]*c[\._-]*)([0-9]*)$',
r'(.*[^a-z])(c[\._-]*)([0-9]+)$',
"_rc": [
r"(.*?)([\._-]*rc[\._-]*)([0-9]*)$",
r"(.*?)([\._-]*c[\._-]*)([0-9]*)$",
r"(.*[^a-z])(c[\._-]*)([0-9]+)$",
],
}
rs_match = None
@ -139,9 +137,9 @@ def gentoo_mangle_version(up_pv):
rev_match = revision_suffixes.search(up_pv)
if rev_match:
pv = up_pv = rev_match.group(1)
replace_me = rev_match.group(2)
# replace_me = rev_match.group(2)
rev = rev_match.group(3)
additional_version = '_p' + rev
additional_version = "_p" + rev
for this_suf in list(suf_matches.keys()):
if rs_match:
@ -156,7 +154,7 @@ def gentoo_mangle_version(up_pv):
if rs_match:
# e.g. 1.0.dev-r1234
major_ver = rs_match.group(1) # 1.0
replace_me = rs_match.group(2) # .dev-r
# replace_me = rs_match.group(2) # .dev-r
rev = rs_match.group(3) # 1234
pv = major_ver + portage_suffix + rev
else:
@ -164,7 +162,7 @@ def gentoo_mangle_version(up_pv):
match = bad_suffixes.search(up_pv)
if match:
suffix = match.groups()[0]
pv = up_pv[: - (len(suffix))]
pv = up_pv[: -(len(suffix))]
pv = pv + additional_version

50
src/euscan/out.py
View File

@ -51,10 +51,12 @@ def progress_bar():
def display():
progress_bar.set(progress_handler.curval, progress_handler.maxval)
progress_handler.display = display
def sigwinch_handler(signum, frame):
lines, progress_bar.term_columns = portage.output.get_term_size()
signal.signal(signal.SIGWINCH, sigwinch_handler)
yield on_progress
@ -75,29 +77,29 @@ def clean_colors(string):
def transform_url(config, cpv, url):
if config['mirror']:
if config["mirror"]:
url = to_mirror(url)
if config['ebuild-uri']:
if config["ebuild-uri"]:
url = to_ebuild_uri(cpv, url)
return url
def to_ebuild_uri(cpv, url):
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
p = '%s-%s' % (pkg, ver)
pvr = '%s%s' % (ver, '-%s' % rev if rev != 'r0' else '')
pf = '%s-%s' % (pkg, pvr)
p = "%s-%s" % (pkg, ver)
pvr = "%s%s" % (ver, "-%s" % rev if rev != "r0" else "")
pf = "%s-%s" % (pkg, pvr)
evars = (
(p, 'P'),
(pkg, 'PN'),
(ver, 'PV'),
(rev, 'PR'),
(pvr, 'PVR'),
(pf, 'PF'),
(cat, 'CATEGORY')
(p, "P"),
(pkg, "PN"),
(ver, "PV"),
(rev, "PR"),
(pvr, "PVR"),
(pf, "PF"),
(cat, "CATEGORY"),
)
for src, dst in evars:
url = url.replace(src, '${%s}' % dst)
url = url.replace(src, "${%s}" % dst)
return url
@ -112,14 +114,14 @@ def load_mirrors():
def from_mirror(url):
if not url.startswith('mirror://'):
if not url.startswith("mirror://"):
return url
if not mirrors_:
load_mirrors()
for mirror_name in mirrors_:
prefix = 'mirror://' + mirror_name
prefix = "mirror://" + mirror_name
if url.startswith(prefix):
return url.replace(prefix, mirrors_[mirror_name][0])
@ -137,7 +139,7 @@ def to_mirror(url):
return "mirror://%s%s%s" % (
mirror_name,
"" if url_part.startswith("/") else "/",
url_part
url_part,
)
return url
@ -146,6 +148,7 @@ class EOutputMem(EOutput):
"""
Override of EOutput, allows to specify an output file for writes
"""
def __init__(self, *args, **kwargs):
super(EOutputMem, self).__init__(*args, **kwargs)
self.out = StringIO()
@ -161,6 +164,7 @@ class EuscanOutput(object):
"""
Class that handles output for euscan
"""
def __init__(self, config):
self.config = config
self.queries = defaultdict(dict)
@ -212,12 +216,10 @@ class EuscanOutput(object):
def result(self, cp, version, urls, handler, confidence):
from euscan.version import get_version_type
cpv = '%s-%s' % (cp, version)
urls = ' '.join(
transform_url(self.config, cpv, url) for url in urls.split()
)
cpv = "%s-%s" % (cp, version)
urls = " ".join(transform_url(self.config, cpv, url) for url in urls.split())
if self.config['format'] in ['json', 'dict']:
if self.config["format"] in ["json", "dict"]:
_curr = self.queries[self.current_query]
_curr["result"].append(
{
@ -225,12 +227,12 @@ class EuscanOutput(object):
"urls": urls.split(),
"handler": handler,
"confidence": confidence,
"type": get_version_type(version)
"type": get_version_type(version),
}
)
else:
if not self.config['quiet']:
print("Upstream Version:", pp.number("%s" % version), end=' ')
if not self.config["quiet"]:
print("Upstream Version:", pp.number("%s" % version), end=" ")
print(pp.path(" %s" % urls))
else:
print(pp.cpv("%s-%s" % (cp, version)) + ":", pp.path(urls))

68
src/euscan/scan.py
View File

@ -1,5 +1,3 @@
import os
import sys
from datetime import datetime
@ -22,7 +20,6 @@ def filter_versions(cp, versions):
filtered = {}
for url, version, handler, confidence in versions:
# Try to keep the most specific urls (determinted by the length)
if version in filtered and len(url) < len(filtered[version]):
continue
@ -31,15 +28,16 @@ def filter_versions(cp, versions):
if version_blacklisted(cp, version):
continue
filtered[version] = {
"url": url,
"handler": handler,
"confidence": confidence
}
filtered[version] = {"url": url, "handler": handler, "confidence": confidence}
return [
(cp, filtered[version]["url"], version, filtered[version]["handler"],
filtered[version]["confidence"])
(
cp,
filtered[version]["url"],
version,
filtered[version]["handler"],
filtered[version]["confidence"],
)
for version in filtered
]
@ -52,7 +50,7 @@ def parse_src_uri(uris):
while uris:
uri = uris.pop()
if '://' not in uri:
if "://" not in uri:
continue
if uris and uris[-1] == "->":
@ -73,16 +71,16 @@ def reload_gentoolkit():
import gentoolkit
# Not used in recent versions
if not hasattr(gentoolkit.package, 'PORTDB'):
if not hasattr(gentoolkit.package, "PORTDB"):
return
PORTDB = portage.db[portage.root]["porttree"].dbapi
if hasattr(gentoolkit.dbapi, 'PORTDB'):
if hasattr(gentoolkit.dbapi, "PORTDB"):
gentoolkit.dbapi.PORTDB = PORTDB
if hasattr(gentoolkit.package, 'PORTDB'):
if hasattr(gentoolkit.package, "PORTDB"):
gentoolkit.package.PORTDB = PORTDB
if hasattr(gentoolkit.query, 'PORTDB'):
if hasattr(gentoolkit.query, "PORTDB"):
gentoolkit.query.PORTDB = PORTDB
@ -104,21 +102,18 @@ def scan_upstream(query, on_progress=None):
)
if not matches:
output.ewarn(
pp.warn("No package matching '%s'" % pp.pkgquery(query))
)
output.ewarn(pp.warn("No package matching '%s'" % pp.pkgquery(query)))
return None
matches = sorted(matches)
pkg = matches.pop()
while '9999' in pkg.version and len(matches):
while "9999" in pkg.version and len(matches):
pkg = matches.pop()
if not pkg:
output.ewarn(
pp.warn("Package '%s' only have a dev version (9999)"
% pp.pkgquery(pkg.cp))
pp.warn("Package '%s' only have a dev version (9999)" % pp.pkgquery(pkg.cp))
)
return None
@ -132,42 +127,34 @@ def scan_upstream(query, on_progress=None):
on_progress(increment=10)
if pkg.cp in BLACKLIST_PACKAGES:
output.ewarn(
pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp))
)
output.ewarn(pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp)))
return None
if not CONFIG['quiet']:
if not CONFIG['format']:
pp.uprint(
" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name()))
)
if not CONFIG["quiet"]:
if not CONFIG["format"]:
pp.uprint(" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name())))
pp.uprint()
else:
output.metadata("overlay", pp.section(pkg.repo_name()))
ebuild_path = pkg.ebuild_path()
if ebuild_path:
output.metadata(
"ebuild", pp.path(os.path.normpath(ebuild_path))
)
output.metadata("ebuild", pp.path(os.path.normpath(ebuild_path)))
uris, homepage, description = pkg.environment(
('SRC_URI', 'HOMEPAGE', 'DESCRIPTION')
("SRC_URI", "HOMEPAGE", "DESCRIPTION")
)
output.metadata("repository", pkg.repo_name())
output.metadata("homepage", homepage)
output.metadata("description", description)
else:
uris = pkg.environment('SRC_URI')
uris = pkg.environment("SRC_URI")
cpv = pkg.cpv
uris = parse_src_uri(uris)
uris_expanded = [
from_mirror(uri) if 'mirror://' in uri else uri for uri in uris
]
uris_expanded = [from_mirror(uri) if "mirror://" in uri else uri for uri in uris]
pkg._uris = uris
pkg._uris_expanded = uris_expanded
@ -187,17 +174,16 @@ def scan_upstream(query, on_progress=None):
is_current_version_stable = is_version_stable(ver)
if len(result) > 0:
if not (CONFIG['format'] or CONFIG['quiet']):
if not (CONFIG["format"] or CONFIG["quiet"]):
print("")
for cp, url, version, handler, confidence in result:
if CONFIG["ignore-pre-release"]:
if not is_version_stable(version):
continue
if CONFIG["ignore-pre-release-if-stable"]:
if is_current_version_stable and \
not is_version_stable(version):
if is_current_version_stable and not is_version_stable(version):
continue
if CONFIG['progress']:
if CONFIG["progress"]:
print("", file=sys.stderr)
output.result(cp, version, url, handler, confidence)

22
src/euscan/version.py
View File

@ -25,24 +25,24 @@ def get_version_type(version):
return types[0] # TODO: consider returning all types
return "release"
# Stolen from pkg_resources, but importing it is not a good idea
component_re = re.compile(r'(\d+ | [a-z]+ | \.| -)', re.VERBOSE)
replace = \
{'pre': 'c', 'preview': 'c', '-': 'final-', 'rc': 'c', 'dev': '@'}.get
component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE)
replace = {"pre": "c", "preview": "c", "-": "final-", "rc": "c", "dev": "@"}.get
def _parse_version_parts(s):
for part in component_re.split(s):
part = replace(part, part)
if not part or part == '.':
if not part or part == ".":
continue
if part[:1] in '0123456789':
if part[:1] in "0123456789":
yield part.zfill(8) # pad for numeric comparison
else:
yield '*' + part
yield "*" + part
yield '*final' # ensure that alpha/beta/candidate are before final
yield "*final" # ensure that alpha/beta/candidate are before final
def parse_version(s):
@ -78,12 +78,12 @@ def parse_version(s):
"""
parts = []
for part in _parse_version_parts(s.lower()):
if part.startswith('*'):
if part < '*final': # remove '-' before a prerelease tag
while parts and parts[-1] == '*final-':
if part.startswith("*"):
if part < "*final": # remove '-' before a prerelease tag
while parts and parts[-1] == "*final-":
parts.pop()
# remove trailing zeros from each series of numeric parts
while parts and parts[-1] == '00000000':
while parts and parts[-1] == "00000000":
parts.pop()
parts.append(part)
return tuple(parts)