diff --git a/bin/euscan b/bin/euscan index c70e686..ff45b79 100755 --- a/bin/euscan +++ b/bin/euscan @@ -35,8 +35,8 @@ from euscan._version import __version__ # Globals -isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() -isatty_stderr = os.environ.get('TERM') != 'dumb' and sys.stderr.isatty() +isatty = os.environ.get("TERM") != "dumb" and sys.stdout.isatty() +isatty_stderr = os.environ.get("TERM") != "dumb" and sys.stderr.isatty() def exit_helper(status): @@ -62,8 +62,7 @@ def setup_signals(): def print_version(): """Output the version info.""" - print("%s (%s) - %s" \ - % (__productname__, __version__, __description__)) + print("%s (%s) - %s" % (__productname__, __version__, __description__)) print() print("Author: %s <%s>" % (__author__, __email__)) print("Copyright 2011 Gentoo Foundation") @@ -78,78 +77,134 @@ def print_usage(_error=None, help=None): if _error: out = sys.stderr - if not _error in ('global-options', 'packages',): + if not _error in ( + "global-options", + "packages", + ): _error = None if not _error and not help: - help = 'all' + help = "all" - if _error in ('global-options',): + if _error in ("global-options",): output.eerror("Wrong option on command line.\n") - if _error in ('packages',): + if _error in ("packages",): output.eerror("You need to specify exactly one package.\n") print(white("Usage:"), file=out) - if _error in ('global-options', 'packages',) or help == 'all': - print(" " + turquoise(__productname__), - yellow("[options]"), - green(" [ [...]]"), file=out) - if _error in ('global-options',) or help == 'all': - print(" " + turquoise(__productname__), - yellow("[--help, --version]"), file=out) + if ( + _error + in ( + "global-options", + "packages", + ) + or help == "all" + ): + print( + " " + turquoise(__productname__), + yellow("[options]"), + green(" [ [...]]"), + file=out, + ) + if _error in ("global-options",) or help == "all": + print(" " + turquoise(__productname__), yellow("[--help, --version]"), file=out) print(file=out) - if _error in ('global-options',) or help: + if _error in ("global-options",) or help: print("Available ", yellow("options") + ":", file=out) - print(yellow(" -C, --nocolor") + - " - turn off colors on output", file=out) - print(yellow(" -q, --quiet") + - " - be as quiet as possible", file=out) - print(yellow(" -h, --help") + - " - display the help screen", file=out) - print(yellow(" -V, --version") + - " - display version info", file=out) + print( + yellow(" -C, --nocolor") + + " - turn off colors on output", + file=out, + ) + print( + yellow(" -q, --quiet") + + " - be as quiet as possible", + file=out, + ) + print( + yellow(" -h, --help") + + " - display the help screen", + file=out, + ) + print( + yellow(" -V, --version") + " - display version info", + file=out, + ) print(file=out) - print(yellow(" -1, --oneshot") + - " - stop as soon as a new version is found", - file=out) - print(yellow(" -b, --brute-force=") + - " - define the brute force " + yellow("") + - " (default: 2)\n" + - " " * 38 + "bigger levels will generate more versions numbers\n" + - " " * 38 + "0 means disabled", file=out) - print(yellow(" -f, --format=") + - " - define the output " + yellow("") + - " (available: json, xml)", file=out) - print(yellow(" -p, --progress") + - " - display a progress bar", file=out) - print(yellow(" -i, --ignore-pre-release") + - " " * 11 + "- Ignore non-stable versions", file=out) - print(yellow(" -I, --ignore-pre-release-if-stable") + - " - Ignore non-stable versions only if current\n" + - " " * 38 + "version is stable", file=out) - print(yellow(" --mirror") + - " - use mirror:// URIs", file=out) - print(yellow(" --ebuild-uri") + - " - use ebuild variables in URIs", file=out) - print(yellow(" --no-handlers") + - " - exclude handlers (comma-separated list)", - file=out) + print( + yellow(" -1, --oneshot") + + " - stop as soon as a new version is found", + file=out, + ) + print( + yellow(" -b, --brute-force=") + + " - define the brute force " + + yellow("") + + " (default: 2)\n" + + " " * 38 + + "bigger levels will generate more versions numbers\n" + + " " * 38 + + "0 means disabled", + file=out, + ) + print( + yellow(" -f, --format=") + + " - define the output " + + yellow("") + + " (available: json, xml)", + file=out, + ) + print( + yellow(" -p, --progress") + " - display a progress bar", + file=out, + ) + print( + yellow(" -i, --ignore-pre-release") + + " " * 11 + + "- Ignore non-stable versions", + file=out, + ) + print( + yellow(" -I, --ignore-pre-release-if-stable") + + " - Ignore non-stable versions only if current\n" + + " " * 38 + + "version is stable", + file=out, + ) + print( + yellow(" --mirror") + " - use mirror:// URIs", + file=out, + ) + print( + yellow(" --ebuild-uri") + + " - use ebuild variables in URIs", + file=out, + ) + print( + yellow(" --no-handlers") + + " - exclude handlers (comma-separated list)", + file=out, + ) print(file=out) - if _error in ('packages',) or help: - print(green(" package") + - " " * 28 + "- the packages (or ebuilds) you want to scan", - file=out) + if _error in ("packages",) or help: + print( + green(" package") + + " " * 28 + + "- the packages (or ebuilds) you want to scan", + file=out, + ) print(file=out) - #print( "More detailed instruction can be found in", - #turquoise("`man %s`" % __productname__), file=out) + # print( "More detailed instruction can be found in", + # turquoise("`man %s`" % __productname__), file=out) class ParseArgsException(Exception): """For parseArgs() -> main() communications.""" + def __init__(self, value): self.value = value @@ -168,92 +223,102 @@ def parse_args(): return_code = True for o, a in opts: if o in ("-h", "--help"): - raise ParseArgsException('help') + raise ParseArgsException("help") elif o in ("-V", "--version"): - raise ParseArgsException('version') + raise ParseArgsException("version") elif o in ("-C", "--nocolor"): - CONFIG['nocolor'] = True + CONFIG["nocolor"] = True pp.output.nocolor() elif o in ("-q", "--quiet"): - CONFIG['quiet'] = True - CONFIG['verbose'] = 0 + CONFIG["quiet"] = True + CONFIG["verbose"] = 0 elif o in ("-1", "--oneshot"): - CONFIG['oneshot'] = True + CONFIG["oneshot"] = True elif o in ("-b", "--brute-force"): - CONFIG['brute-force'] = int(a) - elif o in ("-v", "--verbose") and not CONFIG['quiet']: - CONFIG['verbose'] += 1 + CONFIG["brute-force"] = int(a) + elif o in ("-v", "--verbose") and not CONFIG["quiet"]: + CONFIG["verbose"] += 1 elif o in ("-f", "--format"): - CONFIG['format'] = a - CONFIG['nocolor'] = True + CONFIG["format"] = a + CONFIG["nocolor"] = True pp.output.nocolor() elif o in ("-p", "--progress"): - CONFIG['progress'] = isatty_stderr + CONFIG["progress"] = isatty_stderr elif o in ("--mirror"): - CONFIG['mirror'] = True + CONFIG["mirror"] = True elif o in ("-i", "--ignore-pre-release"): - CONFIG['ignore-pre-release'] = True + CONFIG["ignore-pre-release"] = True elif o in ("-I", "--ignore-pre-release-if-stable"): - CONFIG['ignore-pre-release-if-stable'] = True + CONFIG["ignore-pre-release-if-stable"] = True elif o in ("--ebuild-uri"): - CONFIG['ebuild-uri'] = True + CONFIG["ebuild-uri"] = True elif o in ("--no-handlers"): - CONFIG['handlers-exclude'] = a.split(",") + CONFIG["handlers-exclude"] = a.split(",") else: return_code = False return return_code # here are the different allowed command line options (getopt args) - getopt_options = {'short': {}, 'long': {}} - getopt_options['short']['global'] = "hVCqv1b:f:piI" - getopt_options['long']['global'] = [ - "help", "version", "nocolor", "quiet", "verbose", "oneshot", - "brute-force=", "format=", "progress", "mirror", "ignore-pre-release", - "ignore-pre-release-if-stable", "ebuild-uri", "no-handlers=" + getopt_options = {"short": {}, "long": {}} + getopt_options["short"]["global"] = "hVCqv1b:f:piI" + getopt_options["long"]["global"] = [ + "help", + "version", + "nocolor", + "quiet", + "verbose", + "oneshot", + "brute-force=", + "format=", + "progress", + "mirror", + "ignore-pre-release", + "ignore-pre-release-if-stable", + "ebuild-uri", + "no-handlers=", ] - short_opts = getopt_options['short']['global'] - long_opts = getopt_options['long']['global'] - opts_mode = 'global' + short_opts = getopt_options["short"]["global"] + long_opts = getopt_options["long"]["global"] + opts_mode = "global" # apply getopts to command line, show partial help on failure try: opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts) except: - raise ParseArgsException(opts_mode + '-options') + raise ParseArgsException(opts_mode + "-options") # set options accordingly option_switch(opts) if len(args) < 1: - raise ParseArgsException('packages') + raise ParseArgsException("packages") return args def main(): """Parse command line and execute all actions.""" - CONFIG['nocolor'] = ( - CONFIG['nocolor'] or - (settings["NOCOLOR"] in ('yes', 'true') or not isatty) + CONFIG["nocolor"] = CONFIG["nocolor"] or ( + settings["NOCOLOR"] in ("yes", "true") or not isatty ) - if CONFIG['nocolor']: + if CONFIG["nocolor"]: pp.output.nocolor() # parse command line options and actions try: queries = parse_args() except ParseArgsException as e: - if e.value == 'help': - print_usage(help='all') + if e.value == "help": + print_usage(help="all") exit_helper(0) - elif e.value[:5] == 'help-': + elif e.value[:5] == "help-": print_usage(help=e.value[5:]) exit_helper(0) - elif e.value == 'version': + elif e.value == "version": print_version() exit_helper(0) @@ -261,14 +326,14 @@ def main(): print_usage(e.value) exit_helper(EINVAL) - if CONFIG['verbose'] > 2: + if CONFIG["verbose"] > 2: HTTPConnection.debuglevel = 1 - if not CONFIG["format"] and not CONFIG['quiet']: + if not CONFIG["format"] and not CONFIG["quiet"]: CONFIG["progress"] = False on_progress = None - if CONFIG['progress']: + if CONFIG["progress"]: on_progress_gen = progress_bar() on_progress = next(on_progress_gen) on_progress(maxval=len(queries) * 100, increment=0, label="Working...") @@ -293,35 +358,36 @@ def main(): from os.path import basename # To get the short name output.eerror( - "The short ebuild name '%s' is ambiguous. Please specify" % - basename(pkgs[0]) + - "one of the above fully-qualified ebuild names instead." + "The short ebuild name '%s' is ambiguous. Please specify" + % basename(pkgs[0]) + + "one of the above fully-qualified ebuild names instead." ) exit_helper(1) except GentoolkitException as err: - output.eerror('%s: %s' % (query, str(err))) + output.eerror("%s: %s" % (query, str(err))) exit_helper(1) except Exception as err: import traceback - print ('-' * 60) - traceback.print_exc(file=sys.stderr) - print ('-' * 60) - output.eerror('%s: %s' % (query, str(err))) + print("-" * 60) + traceback.print_exc(file=sys.stderr) + print("-" * 60) + + output.eerror("%s: %s" % (query, str(err))) exit_helper(1) - if not ret and not CONFIG['quiet']: + if not ret and not CONFIG["quiet"]: output.einfo( - "Didn't find any new version, check package's homepage " + - "for more informations" + "Didn't find any new version, check package's homepage " + + "for more informations" ) - if not (CONFIG['format'] or CONFIG['quiet']) and len(queries) > 1: + if not (CONFIG["format"] or CONFIG["quiet"]) and len(queries) > 1: print("") - if CONFIG['progress']: + if CONFIG["progress"]: next(on_progress_gen) print("\n", file=sys.stderr) diff --git a/bin/euscan_patch_metadata b/bin/euscan_patch_metadata index 7bfdfa1..84ee914 100755 --- a/bin/euscan_patch_metadata +++ b/bin/euscan_patch_metadata @@ -28,25 +28,25 @@ def guess_indent_values(before): def guess_for_tags(tags): for tag in tags: for i in [0, 2, 4, 6, 8, 12, 16]: - if '\n%s<%s' % (' ' * i, tag) in before: + if "\n%s<%s" % (" " * i, tag) in before: return i, False for i in [0, 1, 2]: - if '\n%s<%s' % ('\t' * i, tag) in before: + if "\n%s<%s" % ("\t" * i, tag) in before: return i, True return -1, False rindent, tab = guess_for_tags( - ['herd', 'maintainer', 'longdescription', 'use', 'upstream'] + ["herd", "maintainer", "longdescription", "use", "upstream"] ) if rindent == -1: rindent = 2 - rindent_str = ('\t' if tab else ' ') * rindent - indent, tab = guess_for_tags(['watch', 'name', 'email']) + rindent_str = ("\t" if tab else " ") * rindent + indent, tab = guess_for_tags(["watch", "name", "email"]) if indent == -1: indent = rindent * 2 if rindent else 4 - if rindent and rindent_str == '\t': + if rindent and rindent_str == "\t": tab = True - indent_str = ('\t' if tab else ' ') * indent + indent_str = ("\t" if tab else " ") * indent return rindent_str, indent_str @@ -66,7 +66,7 @@ def handle_diff(deb_url): watch_data = "" - fp = gzip.open(temp_deb, 'rb') + fp = gzip.open(temp_deb, "rb") for line in fp: if re.match(r"\+\+\+ .+?/debian/watch", line): fp.readline() # diff lines, don't care @@ -144,8 +144,9 @@ def patch_metadata(package, watch_data, diff=False): data = original # clean watch_data - watch_data = "\n".join([line for line in watch_data.split("\n") - if not line.startswith("#")]) # comments + watch_data = "\n".join( + [line for line in watch_data.split("\n") if not line.startswith("#")] + ) # comments watch_data = watch_data.replace("\\\n", "") # remove backslashes @@ -163,10 +164,7 @@ def patch_metadata(package, watch_data, diff=False): continue # parse watch_line - result = re.match( - r'(?:opts=(?:"([^"]+?)"|([^\s]+?)) )?(.*)', - watch_line - ) + result = re.match(r'(?:opts=(?:"([^"]+?)"|([^\s]+?)) )?(.*)', watch_line) opts_quote, opts, url = result.groups() opts = opts_quote or opts @@ -188,21 +186,27 @@ def patch_metadata(package, watch_data, diff=False): url = " ".join([x for x in url_search.groups() if x is not None]) if opts: - watch_tag = '%s%s' % \ - (indent, version, opts, url) + watch_tag = '%s%s' % ( + indent, + version, + opts, + url, + ) else: - watch_tag = '%s%s' % \ - (indent, version, url) + watch_tag = '%s%s' % (indent, version, url) watch_tags.append(watch_tag) watch_tags = "\n".join(watch_tags) - if '' in data: - data = data.replace('', '\n%s' % watch_tags, 1) + if "" in data: + data = data.replace("", "\n%s" % watch_tags, 1) else: - rep = '%s\n%s\n%s\n' % \ - (rindent, watch_tags, rindent) - data = data.replace('', rep, 1) + rep = "%s\n%s\n%s\n" % ( + rindent, + watch_tags, + rindent, + ) + data = data.replace("", rep, 1) if not diff: return data @@ -214,8 +218,8 @@ def patch_metadata(package, watch_data, diff=False): res = unified_diff( original.splitlines(True), data.splitlines(True), - fromfile=os.path.join('a/', metadata_path), - tofile=os.path.join('b/', metadata_path), + fromfile=os.path.join("a/", metadata_path), + tofile=os.path.join("b/", metadata_path), ) return "".join([x for x in res]) @@ -223,12 +227,12 @@ def patch_metadata(package, watch_data, diff=False): def process_package(query, diff=False): try: matches = Query(query).smart_find( - in_installed=True, - in_porttree=True, - in_overlay=True, - include_masked=True, - show_progress=False, - no_matches_fatal=False, + in_installed=True, + in_porttree=True, + in_overlay=True, + include_masked=True, + show_progress=False, + no_matches_fatal=False, ) except AmbiguousPackageName: logger.error(" Ambiguous package name") @@ -240,7 +244,7 @@ def process_package(query, diff=False): matches = sorted(matches) package = matches.pop() - if '9999' in package.version and len(matches) > 0: + if "9999" in package.version and len(matches) > 0: package = matches.pop() watch_data = get_watch_data(package) @@ -252,16 +256,21 @@ def process_package(query, diff=False): def main(): import optparse + p = optparse.OptionParser( usage="usage: %prog [ [...]]", ) - p.add_option('-d', '--diff', action="store_true", dest="diff", - default=False, - help="Outputs a diff") + p.add_option( + "-d", + "--diff", + action="store_true", + dest="diff", + default=False, + help="Outputs a diff", + ) opts, packages = p.parse_args() - logging.basicConfig(stream=sys.stderr, level=logging.INFO, - format='%(message)s') + logging.basicConfig(stream=sys.stderr, level=logging.INFO, format="%(message)s") for package in packages: logger.info("Processing %s..." % package) @@ -269,5 +278,6 @@ def main(): if result: sys.stdout.write(result) + if __name__ == "__main__": main() diff --git a/src/euscan/__init__.py b/src/euscan/__init__.py index 6990b06..c9127d1 100644 --- a/src/euscan/__init__.py +++ b/src/euscan/__init__.py @@ -9,30 +9,30 @@ from ast import literal_eval CONFIG = { - 'nocolor': False, - 'quiet': False, - 'verbose': 1, - 'debug': False, - 'brute-force': 3, - 'brute-force-recursive': True, - 'brute-force-false-watermark': 50, - 'scan-dir': True, - 'oneshot': True, - 'user-agent': 'escan (http://euscan.iksaif.net)', - 'skip-robots-txt': False, - 'cache': False, - 'format': None, - 'indent': 2, - 'progress': False, - 'mirror': False, - 'ignore-pre-release': False, - 'ignore-pre-release-if-stable': False, - 'ebuild-uri': False, - 'handlers-exclude': [], + "nocolor": False, + "quiet": False, + "verbose": 1, + "debug": False, + "brute-force": 3, + "brute-force-recursive": True, + "brute-force-false-watermark": 50, + "scan-dir": True, + "oneshot": True, + "user-agent": "escan (http://euscan.iksaif.net)", + "skip-robots-txt": False, + "cache": False, + "format": None, + "indent": 2, + "progress": False, + "mirror": False, + "ignore-pre-release": False, + "ignore-pre-release-if-stable": False, + "ebuild-uri": False, + "handlers-exclude": [], } config = configparser.ConfigParser() -config.read(['/etc/euscan.conf', os.path.expanduser('~/.euscan.conf')]) +config.read(["/etc/euscan.conf", os.path.expanduser("~/.euscan.conf")]) if config.has_section("euscan"): for key, value in config.items("euscan"): if key in CONFIG: @@ -41,49 +41,50 @@ if config.has_section("euscan"): BLACKLIST_VERSIONS = [ # Compatibility package for running binaries linked against a # pre gcc 3.4 libstdc++, won't be updated - '>=sys-libs/libstdc++-v3-3.4', + ">=sys-libs/libstdc++-v3-3.4", # Actually older or incorrect - '~app-backup/backup-manager-0.7.15', - '=x11-plugins/wmacpimon-001', + "~app-backup/backup-manager-0.7.15", + "=x11-plugins/wmacpimon-001", ] BLACKLIST_PACKAGES = [ # These kernels are almost dead - 'sys-kernel/xbox-sources', + "sys-kernel/xbox-sources", ] SCANDIR_BLACKLIST_URLS = [ - 'mirror://rubygems/(.*)', # Not browsable - 'mirror://gentoo/(.*)' # Directory too big + "mirror://rubygems/(.*)", # Not browsable + "mirror://gentoo/(.*)", # Directory too big ] BRUTEFORCE_BLACKLIST_PACKAGES = [ # infinite loop any # http://plone.org/products/plonepopoll/releases/*/plonepopoll-2-6-1.tgz # link will work - 'net-zope/plonepopoll' + "net-zope/plonepopoll" ] BRUTEFORCE_BLACKLIST_URLS = [ - 'http://hydra.nixos.org/build/(.*)', # infinite loop + "http://hydra.nixos.org/build/(.*)", # infinite loop # Doesn't respect 404, infinite loop - 'http://www.rennings.net/gentoo/distfiles/(.*)', - 'http://art.gnome.org/download/(.*)', - 'http://barelysufficient.org/~olemarkus/(.*)', - 'http://olemarkus.org/~olemarkus/(.*)', + "http://www.rennings.net/gentoo/distfiles/(.*)", + "http://art.gnome.org/download/(.*)", + "http://barelysufficient.org/~olemarkus/(.*)", + "http://olemarkus.org/~olemarkus/(.*)", ] ROBOTS_TXT_BLACKLIST_DOMAINS = [ - '(.*)sourceforge(.*)', - '(.*)github.com', - '(.*)qt\.nokia\.com(.*)', - '(.*)chromium\.org(.*)', - '(.*)nodejs\.org(.*)', - '(.*)download\.mono-project\.com(.*)', - '(.*)fedorahosted\.org(.*)', - '(.*)download\.tuxfamily\.org(.*)', - '(.*)festvox\.org(.*)', + "(.*)sourceforge(.*)", + "(.*)github.com", + "(.*)qt\.nokia\.com(.*)", + "(.*)chromium\.org(.*)", + "(.*)nodejs\.org(.*)", + "(.*)download\.mono-project\.com(.*)", + "(.*)fedorahosted\.org(.*)", + "(.*)download\.tuxfamily\.org(.*)", + "(.*)festvox\.org(.*)", ] from euscan.out import EuscanOutput + output = EuscanOutput(CONFIG) diff --git a/src/euscan/ebuild.py b/src/euscan/ebuild.py index 8f840cb..f296607 100644 --- a/src/euscan/ebuild.py +++ b/src/euscan/ebuild.py @@ -24,12 +24,10 @@ def package_from_ebuild(ebuild): # since the canonical path returned from os.getcwd() may may be # unusable in cases where the directory stucture is built from # symlinks. - pwd = os.environ.get('PWD', '') + pwd = os.environ.get("PWD", "") if sys.hexversion < 0x3000000: - pwd = _unicode_decode(pwd, encoding=_encodings['content'], - errors='strict') - if pwd and pwd != mycwd and \ - os.path.realpath(pwd) == mycwd: + pwd = _unicode_decode(pwd, encoding=_encodings["content"], errors="strict") + if pwd and pwd != mycwd and os.path.realpath(pwd) == mycwd: mycwd = portage.normalize_path(pwd) ebuild = os.path.join(mycwd, ebuild) @@ -38,22 +36,29 @@ def package_from_ebuild(ebuild): # subdirectories of the base can be built from symlinks (like crossdev # does). ebuild_portdir = os.path.realpath( - os.path.dirname(os.path.dirname(os.path.dirname(ebuild)))) + os.path.dirname(os.path.dirname(os.path.dirname(ebuild))) + ) ebuild = os.path.join(ebuild_portdir, *ebuild.split(os.path.sep)[-3:]) - vdb_path = os.path.join(portage.settings['ROOT'], VDB_PATH) + vdb_path = os.path.join(portage.settings["ROOT"], VDB_PATH) # Make sure that portdb.findname() returns the correct ebuild. - if ebuild_portdir != vdb_path and \ - ebuild_portdir not in portage.portdb.porttrees: + if ebuild_portdir != vdb_path and ebuild_portdir not in portage.portdb.porttrees: if sys.hexversion >= 0x3000000: - os.environ["PORTDIR_OVERLAY"] = \ - os.environ.get("PORTDIR_OVERLAY", "") + \ - " " + _shell_quote(ebuild_portdir) + os.environ["PORTDIR_OVERLAY"] = ( + os.environ.get("PORTDIR_OVERLAY", "") + + " " + + _shell_quote(ebuild_portdir) + ) else: - os.environ["PORTDIR_OVERLAY"] = \ - os.environ.get("PORTDIR_OVERLAY", "") + \ - " " + _unicode_encode(_shell_quote(ebuild_portdir), - encoding=_encodings['content'], errors='strict') + os.environ["PORTDIR_OVERLAY"] = ( + os.environ.get("PORTDIR_OVERLAY", "") + + " " + + _unicode_encode( + _shell_quote(ebuild_portdir), + encoding=_encodings["content"], + errors="strict", + ) + ) portage.close_portdbapi_caches() importlib.reload(portage) diff --git a/src/euscan/handlers/__init__.py b/src/euscan/handlers/__init__.py index 9b53c65..7b9d37b 100644 --- a/src/euscan/handlers/__init__.py +++ b/src/euscan/handlers/__init__.py @@ -6,31 +6,27 @@ from euscan import CONFIG, output from portage.xml.metadata import MetaDataXML -handlers = {'package': [], 'url': [], 'all': {}} +handlers = {"package": [], "url": [], "all": {}} # autoimport all modules in this directory and append them to handlers list for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): - module = loader.find_spec(module_name).loader.load_module(module_name) - if not hasattr(module, 'HANDLER_NAME'): + if not hasattr(module, "HANDLER_NAME"): continue - if hasattr(module, 'scan_url'): - handlers['url'].append(module) - if hasattr(module, 'scan_pkg'): - handlers['package'].append(module) - handlers['all'][module.HANDLER_NAME] = module + if hasattr(module, "scan_url"): + handlers["url"].append(module) + if hasattr(module, "scan_pkg"): + handlers["package"].append(module) + handlers["all"][module.HANDLER_NAME] = module # sort handlers by priority def sort_handlers(handlers): - return sorted( - handlers, - key=lambda handler: handler.PRIORITY, - reverse=True - ) + return sorted(handlers, key=lambda handler: handler.PRIORITY, reverse=True) -handlers['package'] = sort_handlers(handlers['package']) -handlers['url'] = sort_handlers(handlers['url']) + +handlers["package"] = sort_handlers(handlers["package"]) +handlers["url"] = sort_handlers(handlers["url"]) def find_best_handler(kind, pkg, *args): @@ -38,8 +34,9 @@ def find_best_handler(kind, pkg, *args): Find the best handler for the given package """ for handler in handlers[kind]: - if (handler.HANDLER_NAME not in CONFIG["handlers-exclude"] and - handler.can_handle(pkg, *args)): + if handler.HANDLER_NAME not in CONFIG[ + "handlers-exclude" + ] and handler.can_handle(pkg, *args): return handler return None @@ -49,8 +46,8 @@ def find_handlers(kind, names): for name in names: # Does this handler exist, and handle this kind of thing ? (pkg / url) - if name in handlers['all'] and handlers['all'][name] in handlers[kind]: - ret.append(handlers['all'][name]) + if name in handlers["all"] and handlers["all"][name] in handlers[kind]: + ret.append(handlers["all"][name]) return ret @@ -60,17 +57,16 @@ def get_metadata(pkg): pkg_metadata = None - meta_override = os.path.join('metadata', pkg.category, pkg.name, - 'metadata.xml') + meta_override = os.path.join("metadata", pkg.category, pkg.name, "metadata.xml") try: if os.path.exists(meta_override): pkg_metadata = MetaDataXML(meta_override) - output.einfo('Using custom metadata: %s' % meta_override) + output.einfo("Using custom metadata: %s" % meta_override) if not pkg_metadata: pkg_metadata = pkg.metadata except Exception as e: - output.ewarn('Error when fetching metadata: %s' % str(e)) + output.ewarn("Error when fetching metadata: %s" % str(e)) if not pkg_metadata: return {} @@ -79,13 +75,13 @@ def get_metadata(pkg): for upstream in pkg_metadata._xml_tree.findall("upstream"): for node in upstream.findall("watch"): options = dict(node.attrib) - options['data'] = node.text + options["data"] = node.text if "type" in options: - handler = options['type'] + handler = options["type"] else: handler = "url" - options['type'] = "url" + options["type"] = "url" for key in ["versionmangle", "downloadurlmangle"]: value = options.get(key, None) @@ -103,10 +99,10 @@ def get_metadata(pkg): continue if handler in metadata: for i in range(len(metadata[handler])): - if not metadata[handler][i]['data']: - metadata[handler][i]['data'] = node.text + if not metadata[handler][i]["data"]: + metadata[handler][i]["data"] = node.text else: - metadata[handler] = [{'type': handler, 'data': node.text}] + metadata[handler] = [{"type": handler, "data": node.text}] return metadata @@ -145,24 +141,21 @@ def scan_url(pkg, urls, options, on_progress=None): output.einfo("SRC_URI is '%s'" % url) - if '://' not in url: + if "://" not in url: output.einfo("Invalid url '%s'" % url) continue try: - url_handler = find_best_handler('url', pkg, url) + url_handler = find_best_handler("url", pkg, url) if url_handler: for o in options: versions += url_handler.scan_url(pkg, url, o) else: output.eerror("Can't find a suitable handler!") except Exception as e: - output.ewarn( - "Handler failed: [%s] %s" % - (e.__class__.__name__, str(e)) - ) + output.ewarn("Handler failed: [%s] %s" % (e.__class__.__name__, str(e))) - if versions and CONFIG['oneshot']: + if versions and CONFIG["oneshot"]: break if on_progress and progress_available > 0: @@ -178,15 +171,15 @@ def scan(pkg, urls, on_progress=None): in url handling. """ - if not CONFIG['quiet'] and not CONFIG['format']: - sys.stdout.write('\n') + if not CONFIG["quiet"] and not CONFIG["format"]: + sys.stdout.write("\n") metadata = get_metadata(pkg) versions = [] - pkg_handlers = find_handlers('package', list(metadata.keys())) + pkg_handlers = find_handlers("package", list(metadata.keys())) if not pkg_handlers: - pkg_handler = find_best_handler('package', pkg) + pkg_handler = find_best_handler("package", pkg) if pkg_handler: pkg_handlers = [pkg_handler] @@ -201,17 +194,17 @@ def scan(pkg, urls, on_progress=None): def mangle(kind, name, string): - if name not in handlers['all']: + if name not in handlers["all"]: return None - handler = handlers['all'][name] - if not hasattr(handler, 'mangle_%s' % kind): + handler = handlers["all"][name] + if not hasattr(handler, "mangle_%s" % kind): return None - return getattr(handler, 'mangle_%s' % kind)(string) + return getattr(handler, "mangle_%s" % kind)(string) def mangle_url(name, string): - return mangle('url', name, string) + return mangle("url", name, string) def mangle_version(name, string): - return mangle('version', name, string) + return mangle("version", name, string) diff --git a/src/euscan/handlers/berlios.py b/src/euscan/handlers/berlios.py index 89c0f0d..2b852b3 100644 --- a/src/euscan/handlers/berlios.py +++ b/src/euscan/handlers/berlios.py @@ -36,19 +36,13 @@ def scan_url(pkg, url, options): project_page = "http://developer.berlios.de/projects/%s" % project content = urllib.request.urlopen(project_page).read() - project_id = re.search( - r"/project/filelist.php\?group_id=(\d+)", - content - ).group(1) + project_id = re.search(r"/project/filelist.php\?group_id=(\d+)", content).group(1) base_url = ( - "http://developer.berlios.de/project/filelist.php?group_id=%s" % - project_id + "http://developer.berlios.de/project/filelist.php?group_id=%s" % project_id ) - file_pattern = regex_from_template( - filename.replace(ver, "${PV}") - ) + file_pattern = regex_from_template(filename.replace(ver, "${PV}")) result = url_scan(pkg, base_url, file_pattern) diff --git a/src/euscan/handlers/cpan.py b/src/euscan/handlers/cpan.py index 0456ee6..952c460 100644 --- a/src/euscan/handlers/cpan.py +++ b/src/euscan/handlers/cpan.py @@ -13,7 +13,7 @@ _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*") def can_handle(pkg, url=None): - return url and url.startswith('mirror://cpan/') + return url and url.startswith("mirror://cpan/") def guess_package(cp, url): @@ -24,7 +24,7 @@ def guess_package(cp, url): if match: pkg = match.group(1) try: - cp, ver, rev = portage.pkgsplit('fake/' + pkg) + cp, ver, rev = portage.pkgsplit("fake/" + pkg) except: pass @@ -34,7 +34,7 @@ def guess_package(cp, url): def mangle_version(up_pv): - if up_pv.startswith('v'): + if up_pv.startswith("v"): return up_pv[1:] # clean @@ -53,14 +53,14 @@ def mangle_version(up_pv): if len(splitted) == 2: # Split second part is sub-groups part = splitted.pop() for i in range(0, len(part), 3): - splitted.append(part[i:i + 3]) + splitted.append(part[i : i + 3]) if len(splitted) == 2: # add last group if it's missing splitted.append("0") groups = [splitted[0]] for part in splitted[1:-1]: - groups.append(part.ljust(3, "0")) + groups.append(part.ljust(3, "0")) if splitted[-1] == "0": groups.append(splitted[-1]) else: @@ -78,11 +78,11 @@ def mangle_version(up_pv): def cpan_mangle_version(pv): - pos = pv.find('.') + pos = pv.find(".") if pos <= 0: return pv - up_pv = pv.replace('.', '') - up_pv = up_pv[0:pos] + '.' + up_pv[pos:] + up_pv = pv.replace(".", "") + up_pv = up_pv[0:pos] + "." + up_pv[pos:] return up_pv @@ -99,17 +99,17 @@ def scan_url(pkg, url, options): output.einfo("Using CPAN API: %s", remote_pkg) - return scan_pkg(pkg, {'data': remote_pkg}) + return scan_pkg(pkg, {"data": remote_pkg}) def scan_pkg(pkg, options): - remote_pkg = options['data'] + remote_pkg = options["data"] # Defaults to CPAN mangling rules - if 'versionmangle' not in options: - options['versionmangle'] = ['cpan', 'gentoo'] + if "versionmangle" not in options: + options["versionmangle"] = ["cpan", "gentoo"] - url = 'http://search.cpan.org/api/dist/%s' % remote_pkg + url = "http://search.cpan.org/api/dist/%s" % remote_pkg cp, ver, rev = pkg.cp, pkg.version, pkg.revision m_ver = cpan_mangle_version(ver) @@ -128,19 +128,19 @@ def scan_pkg(pkg, options): data = fp.read() data = json.loads(data) - if 'releases' not in data: + if "releases" not in data: return [] ret = [] - for version in data['releases']: - #if version['status'] == 'testing': + for version in data["releases"]: + # if version['status'] == 'testing': # continue - up_pv = version['version'] + up_pv = version["version"] pv = mangling.mangle_version(up_pv, options) - if up_pv.startswith('v'): + if up_pv.startswith("v"): if helpers.version_filtered(cp, ver, pv): continue else: @@ -148,11 +148,11 @@ def scan_pkg(pkg, options): if helpers.version_filtered(cp, m_ver, m_pv, cpan_vercmp): continue - url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % ( - version['cpanid'][0], - version['cpanid'][0:1], - version['cpanid'], - version['archive'] + url = "mirror://cpan/authors/id/%s/%s/%s/%s" % ( + version["cpanid"][0], + version["cpanid"][0:1], + version["cpanid"], + version["archive"], ) url = mangling.mangle_url(url, options) diff --git a/src/euscan/handlers/deb.py b/src/euscan/handlers/deb.py index f0e3d4b..983a58c 100644 --- a/src/euscan/handlers/deb.py +++ b/src/euscan/handlers/deb.py @@ -19,7 +19,7 @@ def can_handle(pkg, url=None): def scan_pkg(pkg, options): cp, ver, rev = portage.pkgsplit(pkg.cpv) - packages_url, package_name = options['data'].strip().split(" ", 1) + packages_url, package_name = options["data"].strip().split(" ", 1) output.einfo("Using Debian Packages: " + packages_url) diff --git a/src/euscan/handlers/freecode.py b/src/euscan/handlers/freecode.py index 94ec649..9d988eb 100644 --- a/src/euscan/handlers/freecode.py +++ b/src/euscan/handlers/freecode.py @@ -17,7 +17,7 @@ def can_handle(pkg, url=None): def scan_pkg(pkg, options): cp, ver, rev = portage.pkgsplit(pkg.cpv) - package = options['data'].strip() + package = options["data"].strip() output.einfo("Using FreeCode handler: " + package) @@ -25,8 +25,7 @@ def scan_pkg(pkg, options): content = str(fp.read()) result = re.findall( - r'([^<]+)' % package, - content + r'([^<]+)' % package, content ) ret = [] @@ -34,15 +33,15 @@ def scan_pkg(pkg, options): pv = mangling.mangle_version(up_pv, options) if helpers.version_filtered(cp, ver, pv): continue - fp = urllib.request.urlopen("http://freecode.com/projects/%s/releases/%s" % - (package, release_id)) + fp = urllib.request.urlopen( + "http://freecode.com/projects/%s/releases/%s" % (package, release_id) + ) content = str(fp.read()) download_page = re.findall(r']*href", data, re.I): results.extend(scan_html(data, url, pattern)) - elif url.startswith('ftp://'): + elif url.startswith("ftp://"): results.extend(scan_ftp(data, url, pattern)) versions = [] @@ -136,8 +140,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url, options): versions.append((path, pv, HANDLER_NAME, confidence)) if steps: - ret = scan_directory_recursive(cp, ver, rev, path, steps, orig_url, - options) + ret = scan_directory_recursive(cp, ver, rev, path, steps, orig_url, options) versions.extend(ret) return versions @@ -160,16 +163,14 @@ def scan_url(pkg, url, options): if ver not in resolved_url: newver = helpers.version_change_end_sep(ver) if newver and newver in resolved_url: - output.einfo( - "Version: using %s instead of %s" % (newver, ver) - ) + output.einfo("Version: using %s instead of %s" % (newver, ver)) ver = newver template = helpers.template_from_url(resolved_url, ver) - if '${' not in template: + if "${" not in template: output.einfo( - "Url doesn't seems to depend on version: %s not found in %s" % - (ver, resolved_url) + "Url doesn't seems to depend on version: %s not found in %s" + % (ver, resolved_url) ) return [] else: @@ -220,10 +221,11 @@ def brute_force(pkg, url): template = helpers.template_from_url(url, ver) - if '${PV}' not in template: + if "${PV}" not in template: output.einfo( - "Url doesn't seems to depend on full version: %s not found in %s" % - (ver, url)) + "Url doesn't seems to depend on full version: %s not found in %s" + % (ver, url) + ) return [] else: output.einfo("Brute forcing: %s" % template) @@ -250,19 +252,15 @@ def brute_force(pkg, url): if not infos: continue - confidence = confidence_score(try_url, url, - minimum=BRUTEFORCE_CONFIDENCE) + confidence = confidence_score(try_url, url, minimum=BRUTEFORCE_CONFIDENCE) result.append([try_url, version, BRUTEFORCE_HANDLER_NAME, confidence]) - if len(result) > CONFIG['brute-force-false-watermark']: - output.einfo( - "Broken server detected ! Skipping brute force." - ) + if len(result) > CONFIG["brute-force-false-watermark"]: + output.einfo("Broken server detected ! Skipping brute force.") return [] if CONFIG["brute-force-recursive"]: - for v in helpers.gen_versions(list(components), - CONFIG["brute-force"]): + for v in helpers.gen_versions(list(components), CONFIG["brute-force"]): if v not in versions and tuple(v) not in done: versions.append(v) diff --git a/src/euscan/handlers/github.py b/src/euscan/handlers/github.py index 6989669..4f87271 100644 --- a/src/euscan/handlers/github.py +++ b/src/euscan/handlers/github.py @@ -12,18 +12,18 @@ PRIORITY = 90 def can_handle(pkg, url=None): - return url and url.startswith('mirror://github/') + return url and url.startswith("mirror://github/") def guess_package(cp, url): - match = re.search('^mirror://github/(.*?)/(.*?)/(.*)$', url) + match = re.search("^mirror://github/(.*?)/(.*?)/(.*)$", url) - assert(match) + assert match return (match.group(1), match.group(2), match.group(3)) def scan_url(pkg, url, options): - 'http://developer.github.com/v3/repos/downloads/' + "http://developer.github.com/v3/repos/downloads/" user, project, filename = guess_package(pkg.cpv, url) @@ -35,25 +35,27 @@ def scan_url(pkg, url, options): # now create a filename-matching regexp # XXX: supposedly replace first with (?P...) # and remaining ones with (?P=foo) - fnre = re.compile('^%s$' % \ - re.escape(filename).replace(re.escape(ver), '(.*?)')) + fnre = re.compile("^%s$" % re.escape(filename).replace(re.escape(ver), "(.*?)")) - output.einfo("Using github API for: project=%s user=%s filename=%s" % \ - (project, user, filename)) + output.einfo( + "Using github API for: project=%s user=%s filename=%s" + % (project, user, filename) + ) - dlreq = urllib.request.urlopen('https://api.github.com/repos/%s/%s/downloads' % \ - (user, project)) + dlreq = urllib.request.urlopen( + "https://api.github.com/repos/%s/%s/downloads" % (user, project) + ) dls = json.load(dlreq) ret = [] for dl in dls: - m = fnre.match(dl['name']) + m = fnre.match(dl["name"]) if m: pv = mangling.mangle_version(m.group(1), options) if helpers.version_filtered(cp, ver, pv): continue - url = mangling.mangle_url(dl['html_url'], options) + url = mangling.mangle_url(dl["html_url"], options) ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret diff --git a/src/euscan/handlers/gnome.py b/src/euscan/handlers/gnome.py index ada57b6..3464120 100644 --- a/src/euscan/handlers/gnome.py +++ b/src/euscan/handlers/gnome.py @@ -16,15 +16,15 @@ HANDLER_NAME = "gnome" CONFIDENCE = 100 PRIORITY = 90 -GNOME_URL_SOURCE = 'http://ftp.gnome.org/pub/GNOME/sources' +GNOME_URL_SOURCE = "http://ftp.gnome.org/pub/GNOME/sources" def can_handle(_pkg, url=None): - return url and url.startswith('mirror://gnome/') + return url and url.startswith("mirror://gnome/") def guess_package(cp, url): - match = re.search('mirror://gnome/sources/([^/]+)/.*', url) + match = re.search("mirror://gnome/sources/([^/]+)/.*", url) if match: return match.group(1) @@ -34,27 +34,27 @@ def guess_package(cp, url): def scan_url(pkg, url, options): - 'http://ftp.gnome.org/pub/GNOME/sources/' + "http://ftp.gnome.org/pub/GNOME/sources/" package = { - 'data': guess_package(pkg.cpv, url), - 'type': 'gnome', + "data": guess_package(pkg.cpv, url), + "type": "gnome", } return scan_pkg(pkg, package) def scan_pkg(pkg, options): - package = options['data'] + package = options["data"] output.einfo("Using Gnome json cache: " + package) - fp = urllib.request.urlopen('/'.join([GNOME_URL_SOURCE, package, 'cache.json'])) + fp = urllib.request.urlopen("/".join([GNOME_URL_SOURCE, package, "cache.json"])) content = fp.read() fp.close() - cache = json.loads(content, encoding='ascii') + cache = json.loads(content, encoding="ascii") if cache[0] != 4: - output.eerror('Unknow cache format detected') + output.eerror("Unknow cache format detected") return [] versions = cache[2][package] @@ -72,13 +72,12 @@ def scan_pkg(pkg, options): if helpers.version_filtered(cp, ver, pv): continue up_files = cache[1][package][up_pv] - for tarball_comp in ('tar.xz', 'tar.bz2', 'tar.gz'): + for tarball_comp in ("tar.xz", "tar.bz2", "tar.gz"): if tarball_comp in up_files: - url = '/'.join([GNOME_URL_SOURCE, package, - up_files[tarball_comp]]) + url = "/".join([GNOME_URL_SOURCE, package, up_files[tarball_comp]]) break else: - output.ewarn('No tarball for release %s' % up_pv) + output.ewarn("No tarball for release %s" % up_pv) ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret diff --git a/src/euscan/handlers/google_code.py b/src/euscan/handlers/google_code.py index d6c5bd1..d01993d 100644 --- a/src/euscan/handlers/google_code.py +++ b/src/euscan/handlers/google_code.py @@ -23,6 +23,7 @@ def can_handle(pkg, url=None): return re.match(package_name_regex, url) + def scan_url(pkg, url, options): output.einfo("Using Google Code handler") @@ -31,9 +32,7 @@ def scan_url(pkg, url, options): package_name = re.match(package_name_regex, url).group(1) base_url = "http://code.google.com/p/%s/downloads/list" % package_name - file_pattern = regex_from_template( - url.split("/")[-1].replace(ver, "${PV}") - ) + file_pattern = regex_from_template(url.split("/")[-1].replace(ver, "${PV}")) result = url_scan(pkg, base_url, file_pattern) diff --git a/src/euscan/handlers/kde.py b/src/euscan/handlers/kde.py index 90738de..78457d0 100644 --- a/src/euscan/handlers/kde.py +++ b/src/euscan/handlers/kde.py @@ -6,14 +6,14 @@ HANDLER_NAME = "kde" def can_handle(pkg, url): - return url and url.startswith('mirror://kde/') + return url and url.startswith("mirror://kde/") def clean_results(results): ret = [] for path, version, _, confidence in results: - if version == '5SUMS': + if version == "5SUMS": continue ret.append((path, version, HANDLER_NAME, confidence)) @@ -23,16 +23,15 @@ def clean_results(results): def scan_url(pkg, url, options): results = generic.scan(pkg.cpv, url) - if generic.startswith('mirror://kde/unstable/'): - url = generic.replace('mirror://kde/unstable/', 'mirror://kde/stable/') + if generic.startswith("mirror://kde/unstable/"): + url = generic.replace("mirror://kde/unstable/", "mirror://kde/stable/") results += generic.scan(pkg.cpv, url) if not results: # if nothing was found go brute forcing results = generic.brute_force(pkg.cpv, url) - if generic.startswith('mirror://kde/unstable/'): - url = generic.replace('mirror://kde/unstable/', - 'mirror://kde/stable/') + if generic.startswith("mirror://kde/unstable/"): + url = generic.replace("mirror://kde/unstable/", "mirror://kde/stable/") results += generic.brute_force(pkg.cpv, url) return clean_results(results) diff --git a/src/euscan/handlers/pear.py b/src/euscan/handlers/pear.py index d9d1548..f0b47b2 100644 --- a/src/euscan/handlers/pear.py +++ b/src/euscan/handlers/pear.py @@ -6,7 +6,8 @@ PRIORITY = 90 def can_handle(pkg, url=None): - return url and url.startswith('http://%s.php.net/get/' % HANDLER_NAME) + return url and url.startswith("http://%s.php.net/get/" % HANDLER_NAME) + scan_url = php.scan_url scan_pkg = php.scan_pkg diff --git a/src/euscan/handlers/pecl.py b/src/euscan/handlers/pecl.py index cf372d2..3265a86 100644 --- a/src/euscan/handlers/pecl.py +++ b/src/euscan/handlers/pecl.py @@ -4,8 +4,10 @@ HANDLER_NAME = "pecl" CONFIDENCE = 100 PRIORITY = 90 + def can_handle(pkg, url=None): - return url and url.startswith('http://%s.php.net/get/' % HANDLER_NAME) + return url and url.startswith("http://%s.php.net/get/" % HANDLER_NAME) + scan_url = php.scan_url scan_pkg = php.scan_pkg diff --git a/src/euscan/handlers/php.py b/src/euscan/handlers/php.py index 10d863e..db0b84a 100644 --- a/src/euscan/handlers/php.py +++ b/src/euscan/handlers/php.py @@ -9,11 +9,13 @@ HANDLER_NAME = "php" CONFIDENCE = 100 PRIORITY = 90 + def can_handle(pkg, url=None): return False + def guess_package_and_channel(cp, url): - match = re.search('http://(.*)\.php\.net/get/(.*)-(.*).tgz', url) + match = re.search("http://(.*)\.php\.net/get/(.*)-(.*).tgz", url) if match: host = match.group(1) @@ -26,15 +28,16 @@ def guess_package_and_channel(cp, url): def scan_url(pkg, url, options): package, channel = guess_package_and_channel(pkg.cp, url) - return scan_pkg(pkg, {'type' : channel, 'data' : package }) + return scan_pkg(pkg, {"type": channel, "data": package}) + def scan_pkg(pkg, options): cp, ver, rev = pkg.cp, pkg.version, pkg.revision - package = options['data'] - channel = options['type'] + package = options["data"] + channel = options["type"] - url = 'http://%s.php.net/rest/r/%s/allreleases.xml' % (channel, package.lower()) + url = "http://%s.php.net/rest/r/%s/allreleases.xml" % (channel, package.lower()) output.einfo("Using: " + url) @@ -61,7 +64,7 @@ def scan_pkg(pkg, options): if helpers.version_filtered(cp, ver, pv): continue - url = 'http://%s.php.net/get/%s-%s.tgz' % (channel, package, up_pv) + url = "http://%s.php.net/get/%s-%s.tgz" % (channel, package, up_pv) url = mangling.mangle_url(url, options) ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) diff --git a/src/euscan/handlers/pypi.py b/src/euscan/handlers/pypi.py index ae6bbb5..9e9043a 100644 --- a/src/euscan/handlers/pypi.py +++ b/src/euscan/handlers/pypi.py @@ -11,11 +11,11 @@ PRIORITY = 90 def can_handle(pkg, url=None): - return url and url.startswith('mirror://pypi/') + return url and url.startswith("mirror://pypi/") def guess_package(cp, url): - match = re.search('mirror://pypi/\w+/(.*)/.*', url) + match = re.search("mirror://pypi/\w+/(.*)/.*", url) if match: return match.group(1) @@ -25,18 +25,18 @@ def guess_package(cp, url): def scan_url(pkg, url, options): - 'http://wiki.python.org/moin/PyPiXmlRpc' + "http://wiki.python.org/moin/PyPiXmlRpc" package = guess_package(pkg.cpv, url) - return scan_pkg(pkg, {'data': package}) + return scan_pkg(pkg, {"data": package}) def scan_pkg(pkg, options): - package = options['data'] + package = options["data"] output.einfo("Using PyPi XMLRPC: " + package) - client = xmlrpc.client.ServerProxy('https://pypi.python.org/pypi') + client = xmlrpc.client.ServerProxy("https://pypi.python.org/pypi") versions = client.package_releases(package) if not versions: @@ -52,7 +52,6 @@ def scan_pkg(pkg, options): if helpers.version_filtered(cp, ver, pv): continue urls = client.release_urls(package, up_pv) - urls = " ".join([mangling.mangle_url(infos['url'], options) - for infos in urls]) + urls = " ".join([mangling.mangle_url(infos["url"], options) for infos in urls]) ret.append((urls, pv, HANDLER_NAME, CONFIDENCE)) return ret diff --git a/src/euscan/handlers/rubygems.py b/src/euscan/handlers/rubygems.py index e9a8b08..8f19d97 100644 --- a/src/euscan/handlers/rubygems.py +++ b/src/euscan/handlers/rubygems.py @@ -11,13 +11,13 @@ PRIORITY = 90 def can_handle(pkg, url=None): - return url and url.startswith('mirror://rubygems/') + return url and url.startswith("mirror://rubygems/") def guess_gem(cpv, url): - match = re.search('mirror://rubygems/(.*).gem', url) + match = re.search("mirror://rubygems/(.*).gem", url) if match: - cpv = 'fake/%s' % match.group(1) + cpv = "fake/%s" % match.group(1) ret = portage.pkgsplit(cpv) if not ret: @@ -30,23 +30,22 @@ def guess_gem(cpv, url): def scan_url(pkg, url, options): - 'http://guides.rubygems.org/rubygems-org-api/#gemversion' + "http://guides.rubygems.org/rubygems-org-api/#gemversion" gem = guess_gem(pkg.cpv, url) if not gem: - output.eerror("Can't guess gem name using %s and %s" % \ - (pkg.cpv, url)) + output.eerror("Can't guess gem name using %s and %s" % (pkg.cpv, url)) return [] output.einfo("Using RubyGem API: %s" % gem) - return scan_pkg(pkg, {'data': gem}) + return scan_pkg(pkg, {"data": gem}) def scan_pkg(pkg, options): - gem = options['data'] - url = 'http://rubygems.org/api/v1/versions/%s.json' % gem + gem = options["data"] + url = "http://rubygems.org/api/v1/versions/%s.json" % gem try: fp = helpers.urlopen(url) @@ -65,11 +64,11 @@ def scan_pkg(pkg, options): ret = [] for version in versions: - up_pv = version['number'] + up_pv = version["number"] pv = mangling.mangle_version(up_pv, options) if helpers.version_filtered(cp, ver, pv): continue - url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv) + url = "http://rubygems.org/gems/%s-%s.gem" % (gem, up_pv) url = mangling.mangle_url(url, options) ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret diff --git a/src/euscan/handlers/sourceforge.py b/src/euscan/handlers/sourceforge.py index 0508289..01caed6 100644 --- a/src/euscan/handlers/sourceforge.py +++ b/src/euscan/handlers/sourceforge.py @@ -28,14 +28,11 @@ def scan_url(pkg, url, options): cp, ver, rev = portage.pkgsplit(pkg.cpv) project, filename = re.search( - "mirror://sourceforge/([^/]+)/(?:.*/)?([^/]+)", - url + "mirror://sourceforge/([^/]+)/(?:.*/)?([^/]+)", url ).groups() base_url = "http://qa.debian.org/watch/sf.php/%s" % project - file_pattern = regex_from_template( - filename.replace(ver, "${PV}") - ) + file_pattern = regex_from_template(filename.replace(ver, "${PV}")) result = url_scan(pkg, base_url, file_pattern) diff --git a/src/euscan/handlers/url.py b/src/euscan/handlers/url.py index d4a30aa..41d380d 100644 --- a/src/euscan/handlers/url.py +++ b/src/euscan/handlers/url.py @@ -33,7 +33,7 @@ def handle_directory_patterns(base, file_pattern): i += 1 basedir = "/".join(basedir) directory_pattern = splitted[i] - final = "/".join(splitted[i + 1:]) + final = "/".join(splitted[i + 1 :]) try: fp = helpers.urlopen(basedir) @@ -52,15 +52,14 @@ def handle_directory_patterns(base, file_pattern): else: scan_data = generic.scan_html(data, basedir, directory_pattern) - return [("/".join((basedir, path, final)), file_pattern) - for _, path in scan_data] + return [("/".join((basedir, path, final)), file_pattern) for _, path in scan_data] def read_options(options): try: - base, file_pattern = options['data'].split(" ")[:2] + base, file_pattern = options["data"].split(" ")[:2] except ValueError: - base, file_pattern = options['data'], None + base, file_pattern = options["data"], None # the file pattern can be in the base url pattern_regex = r"/([^/]*\([^/]*\)[^/]*)$" @@ -70,9 +69,7 @@ def read_options(options): base = base.replace(file_pattern, "") # handle sf.net specially - base = base.replace( - "http://sf.net/", "http://qa.debian.org/watch/sf.php/" - ) + base = base.replace("http://sf.net/", "http://qa.debian.org/watch/sf.php/") return base, file_pattern diff --git a/src/euscan/helpers.py b/src/euscan/helpers.py index 831bc51..c69f620 100644 --- a/src/euscan/helpers.py +++ b/src/euscan/helpers.py @@ -21,18 +21,17 @@ from euscan.version import parse_version def htop_vercmp(a, b): def fixver(v): - if v in ['0.11', '0.12', '0.13']: - v = '0.1.' + v[3:] + if v in ["0.11", "0.12", "0.13"]: + v = "0.1." + v[3:] return v return simple_vercmp(fixver(a), fixver(b)) -VERSION_CMP_PACKAGE_QUIRKS = { - 'sys-process/htop': htop_vercmp -} -_v_end = r'(?:(?:-|_)(?:pre|p|beta|b|alpha|a|rc|r)\d*)' -_v = r'((?:\d+)(?:(?:\.\d+)*)(?:[a-zA-Z]*?)(?:' + _v_end + '*))' +VERSION_CMP_PACKAGE_QUIRKS = {"sys-process/htop": htop_vercmp} + +_v_end = r"(?:(?:-|_)(?:pre|p|beta|b|alpha|a|rc|r)\d*)" +_v = r"((?:\d+)(?:(?:\.\d+)*)(?:[a-zA-Z]*?)(?:" + _v_end + "*))" def cast_int_components(version): @@ -75,15 +74,15 @@ def version_is_nightly(a, b): b = parse_version(b) # Try to skip nightly builds when not wanted (www-apps/moodle) - if len(a) != len(b) and len(b) == 2 and len(b[0]) == len('yyyymmdd'): - if b[0][:4] != '0000': + if len(a) != len(b) and len(b) == 2 and len(b[0]) == len("yyyymmdd"): + if b[0][:4] != "0000": return True return False def version_blacklisted(cp, version): rule = None - cpv = '%s-%s' % (cp, version) + cpv = "%s-%s" % (cp, version) # Check that the generated cpv can be used by portage if not portage.versions.catpkgsplit(cpv): @@ -104,10 +103,10 @@ def version_change_end_sep(version): if not match: return None end = match.group(1) - if end[0] == '_': - newend = end.replace('_', '-') - elif end[0] == '-': - newend = end.replace('-', '_') + if end[0] == "_": + newend = end.replace("_", "-") + elif end[0] == "-": + newend = end.replace("-", "_") else: return None return version.replace(end, newend) @@ -135,17 +134,17 @@ def generate_templates_vars(version): var = [] for j in range(i): ver.append(str(part[j])) - var.append('${%d}' % j) + var.append("${%d}" % j) ret.append((".".join(ver), ".".join(var))) - ret.append((version, '${PV}')) + ret.append((version, "${PV}")) ret.reverse() return ret def template_from_url(url, version): - prefix, chunks = url.split('://') - chunks = chunks.split('/') + prefix, chunks = url.split("://") + chunks = chunks.split("/") for i in range(len(chunks)): chunk = chunks[i] @@ -162,9 +161,9 @@ def template_from_url(url, version): def url_from_template(url, version): components = split_version(version) - url = url.replace('${PV}', version) + url = url.replace("${PV}", version) for i in range(len(components)): - url = url.replace('${%d}' % i, str(components[i])) + url = url.replace("${%d}" % i, str(components[i])) return url @@ -172,8 +171,8 @@ def url_from_template(url, version): # Stolen from distutils.LooseVersion # Used for brute force to increment the version def split_version(version): - component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) - components = [x for x in component_re.split(version) if x and x != '.'] + component_re = re.compile(r"(\d+ | [a-z]+ | \.)", re.VERBOSE) + components = [x for x in component_re.split(version) if x and x != "."] for i in range(len(components)): try: components[i] = int(components[i]) @@ -229,7 +228,7 @@ def gen_versions(components, level): def timeout_for_url(url): - if 'sourceforge' in url: + if "sourceforge" in url: timeout = 15 else: timeout = 5 @@ -246,7 +245,7 @@ rpcache = {} def urlallowed(url): - if CONFIG['skip-robots-txt']: + if CONFIG["skip-robots-txt"]: return True protocol, domain = urllib.parse.urlparse(url)[:2] @@ -255,15 +254,15 @@ def urlallowed(url): if re.match(bd, domain): return True - for d in ['sourceforge', 'berlios', 'github.com']: + for d in ["sourceforge", "berlios", "github.com"]: if d in domain: return True - if protocol == 'ftp': + if protocol == "ftp": return True - baseurl = '%s://%s' % (protocol, domain) - robotsurl = urllib.parse.urljoin(baseurl, 'robots.txt') + baseurl = "%s://%s" % (protocol, domain) + robotsurl = urllib.parse.urljoin(baseurl, "robots.txt") if baseurl in rpcache: rp = rpcache[baseurl] @@ -283,7 +282,7 @@ def urlallowed(url): setdefaulttimeout(timeout) - return rp.can_fetch(CONFIG['user-agent'], url) if rp else True + return rp.can_fetch(CONFIG["user-agent"], url) if rp else True def urlopen(url, timeout=None, verb="GET"): @@ -294,23 +293,24 @@ def urlopen(url, timeout=None, verb="GET"): if not timeout: timeout = timeout_for_url(url) - if verb == 'GET': + if verb == "GET": request = urllib.request.Request(url) - elif verb == 'HEAD': + elif verb == "HEAD": request = HeadRequest(url) else: return None - request.add_header('User-Agent', CONFIG['user-agent']) + request.add_header("User-Agent", CONFIG["user-agent"]) handlers = [] - if CONFIG['cache']: + if CONFIG["cache"]: from cache import CacheHandler - handlers.append(CacheHandler(CONFIG['cache'])) - if CONFIG['verbose']: - debuglevel = CONFIG['verbose'] - 1 + handlers.append(CacheHandler(CONFIG["cache"])) + + if CONFIG["verbose"]: + debuglevel = CONFIG["verbose"] - 1 handlers.append(urllib.request.HTTPHandler(debuglevel=debuglevel)) opener = urllib.request.build_opener(*handlers) @@ -330,7 +330,7 @@ def tryurl(fileurl, template): try: basename = os.path.basename(fileurl) - fp = urlopen(fileurl, verb='HEAD') + fp = urlopen(fileurl, verb="HEAD") if not fp: euscan.output.eend(errno.EPERM) return None @@ -339,16 +339,17 @@ def tryurl(fileurl, template): # Some URLs return Content-disposition with different filename # Disable check for now (I have no seen false positives) - #if 'Content-disposition' in headers and \ + # if 'Content-disposition' in headers and \ # basename not in headers['Content-disposition']: # result = None - if 'Content-Length' in headers and headers['Content-Length'] == '0': + if "Content-Length" in headers and headers["Content-Length"] == "0": result = None - elif 'Content-Type' in headers and \ - 'text/html' in headers['Content-Type']: + elif "Content-Type" in headers and "text/html" in headers["Content-Type"]: result = None - elif 'Content-Type' in headers and \ - 'application/x-httpd-php' in headers['Content-Type']: + elif ( + "Content-Type" in headers + and "application/x-httpd-php" in headers["Content-Type"] + ): result = None elif fp.geturl() != fileurl: regex = regex_from_template(template) @@ -356,8 +357,9 @@ def tryurl(fileurl, template): basename2 = os.path.basename(fp.geturl()) # Redirect to another (earlier?) version - if basename != basename2 and (re.match(regex, fp.geturl()) or \ - re.match(baseregex, basename2)): + if basename != basename2 and ( + re.match(regex, fp.geturl()) or re.match(baseregex, basename2) + ): result = None if result: @@ -378,33 +380,33 @@ def regex_from_template(template): regexp = re.escape(template) # Unescape specific stuff - regexp = regexp.replace('\$\{', '${') - regexp = regexp.replace('\}', '}') - regexp = regexp.replace('}\.$', '}.$') + regexp = regexp.replace("\$\{", "${") + regexp = regexp.replace("\}", "}") + regexp = regexp.replace("}\.$", "}.$") # Replace ${\d+} - #regexp = regexp.replace('${0}', r'([\d]+?)') - regexp = re.sub(r'(\$\{\d+\}(\.?))+', r'([\\w\.]+?)', regexp) + # regexp = regexp.replace('${0}', r'([\d]+?)') + regexp = re.sub(r"(\$\{\d+\}(\.?))+", r"([\\w\.]+?)", regexp) - #regexp = re.sub(r'(\$\{\d+\}\.?)+', r'([\w]+?)', regexp) - #regexp = re.sub(r'(\$\{\d+\}\.+)+', '(.+?)\.', regexp) - #regexp = re.sub(r'(\$\{\d+\})+', '(.+?)', regexp) + # regexp = re.sub(r'(\$\{\d+\}\.?)+', r'([\w]+?)', regexp) + # regexp = re.sub(r'(\$\{\d+\}\.+)+', '(.+?)\.', regexp) + # regexp = re.sub(r'(\$\{\d+\})+', '(.+?)', regexp) # Full version - regexp = regexp.replace('${PV}', _v) + regexp = regexp.replace("${PV}", _v) # End - regexp = regexp + r'/?$' + regexp = regexp + r"/?$" return regexp def basedir_from_template(template): - idx = template.find('${') + idx = template.find("${") if idx == -1: return template - idx = template[0:idx].rfind('/') + idx = template[0:idx].rfind("/") if idx == -1: return "" @@ -412,15 +414,15 @@ def basedir_from_template(template): def generate_scan_paths(url): - prefix, chunks = url.split('://') - chunks = chunks.split('/') + prefix, chunks = url.split("://") + chunks = chunks.split("/") steps = [] path = prefix + ":/" for chunk in chunks: - if '${' in chunk: - steps.append((path, '^(?:|.*/)' + regex_from_template(chunk))) + if "${" in chunk: + steps.append((path, "^(?:|.*/)" + regex_from_template(chunk))) path = "" else: path += "/" @@ -444,7 +446,7 @@ def parse_mirror(uri): return None mirrorname = uri[9:eidx] - path = uri[eidx + 1:] + path = uri[eidx + 1 :] if mirrorname in mirrors: mirrors = mirrors[mirrorname] diff --git a/src/euscan/mangling.py b/src/euscan/mangling.py index 67d3314..c50ea50 100644 --- a/src/euscan/mangling.py +++ b/src/euscan/mangling.py @@ -31,11 +31,11 @@ def apply_mangling_rules(kind, rules, string): ret = None # First try handlers rules - if rule == 'gentoo' and kind == 'versionmangle': + if rule == "gentoo" and kind == "versionmangle": ret = gentoo_mangle_version(string) - elif kind == 'downloadurlmangle': + elif kind == "downloadurlmangle": ret = euscan.handlers.mangle_url(rule, string) - elif kind == 'versionmangle': + elif kind == "versionmangle": ret = euscan.handlers.mangle_version(rule, string) if ret is not None: # Use return value as new string if not None @@ -48,13 +48,13 @@ def apply_mangling_rules(kind, rules, string): def mangle_version(up_pv, options): # Default rule is gentoo when empty - if 'versionmangle' not in options or not options['versionmangle']: - options['versionmangle'] = ['gentoo'] - return apply_mangling_rules('versionmangle', options, up_pv) + if "versionmangle" not in options or not options["versionmangle"]: + options["versionmangle"] = ["gentoo"] + return apply_mangling_rules("versionmangle", options, up_pv) def mangle_url(url, options): - return apply_mangling_rules('downloadurlmangle', options, url) + return apply_mangling_rules("downloadurlmangle", options, url) # Stolen from g-pypi @@ -107,30 +107,28 @@ def gentoo_mangle_version(up_pv): number of match.groups every time to simplify the code """ - bad_suffixes = re.compile( - r'((?:[._-]*)(?:dev|devel|final|stable|snapshot)$)', re.I) - revision_suffixes = re.compile( - r'(.*?)([\._-]*(?:r|patch|p)[\._-]*)([0-9]*)$', re.I) + bad_suffixes = re.compile(r"((?:[._-]*)(?:dev|devel|final|stable|snapshot)$)", re.I) + revision_suffixes = re.compile(r"(.*?)([\._-]*(?:r|patch|p)[\._-]*)([0-9]*)$", re.I) suf_matches = { - '_pre': [ - r'(.*?)([\._-]*dev[\._-]*r?)([0-9]+)$', - r'(.*?)([\._-]*(?:pre|preview)[\._-]*)([0-9]*)$', - ], - '_alpha': [ - r'(.*?)([\._-]*(?:alpha|test)[\._-]*)([0-9]*)$', - r'(.*?)([\._-]*a[\._-]*)([0-9]*)$', - r'(.*[^a-z])(a)([0-9]*)$', - ], - '_beta': [ - r'(.*?)([\._-]*beta[\._-]*)([0-9]*)$', - r'(.*?)([\._-]*b)([0-9]*)$', - r'(.*[^a-z])(b)([0-9]*)$', - ], - '_rc': [ - r'(.*?)([\._-]*rc[\._-]*)([0-9]*)$', - r'(.*?)([\._-]*c[\._-]*)([0-9]*)$', - r'(.*[^a-z])(c[\._-]*)([0-9]+)$', - ], + "_pre": [ + r"(.*?)([\._-]*dev[\._-]*r?)([0-9]+)$", + r"(.*?)([\._-]*(?:pre|preview)[\._-]*)([0-9]*)$", + ], + "_alpha": [ + r"(.*?)([\._-]*(?:alpha|test)[\._-]*)([0-9]*)$", + r"(.*?)([\._-]*a[\._-]*)([0-9]*)$", + r"(.*[^a-z])(a)([0-9]*)$", + ], + "_beta": [ + r"(.*?)([\._-]*beta[\._-]*)([0-9]*)$", + r"(.*?)([\._-]*b)([0-9]*)$", + r"(.*[^a-z])(b)([0-9]*)$", + ], + "_rc": [ + r"(.*?)([\._-]*rc[\._-]*)([0-9]*)$", + r"(.*?)([\._-]*c[\._-]*)([0-9]*)$", + r"(.*[^a-z])(c[\._-]*)([0-9]+)$", + ], } rs_match = None pv = up_pv @@ -139,9 +137,9 @@ def gentoo_mangle_version(up_pv): rev_match = revision_suffixes.search(up_pv) if rev_match: pv = up_pv = rev_match.group(1) - replace_me = rev_match.group(2) + # replace_me = rev_match.group(2) rev = rev_match.group(3) - additional_version = '_p' + rev + additional_version = "_p" + rev for this_suf in list(suf_matches.keys()): if rs_match: @@ -156,7 +154,7 @@ def gentoo_mangle_version(up_pv): if rs_match: # e.g. 1.0.dev-r1234 major_ver = rs_match.group(1) # 1.0 - replace_me = rs_match.group(2) # .dev-r + # replace_me = rs_match.group(2) # .dev-r rev = rs_match.group(3) # 1234 pv = major_ver + portage_suffix + rev else: @@ -164,7 +162,7 @@ def gentoo_mangle_version(up_pv): match = bad_suffixes.search(up_pv) if match: suffix = match.groups()[0] - pv = up_pv[: - (len(suffix))] + pv = up_pv[: -(len(suffix))] pv = pv + additional_version diff --git a/src/euscan/out.py b/src/euscan/out.py index dc4face..071588f 100644 --- a/src/euscan/out.py +++ b/src/euscan/out.py @@ -51,10 +51,12 @@ def progress_bar(): def display(): progress_bar.set(progress_handler.curval, progress_handler.maxval) + progress_handler.display = display def sigwinch_handler(signum, frame): lines, progress_bar.term_columns = portage.output.get_term_size() + signal.signal(signal.SIGWINCH, sigwinch_handler) yield on_progress @@ -75,29 +77,29 @@ def clean_colors(string): def transform_url(config, cpv, url): - if config['mirror']: + if config["mirror"]: url = to_mirror(url) - if config['ebuild-uri']: + if config["ebuild-uri"]: url = to_ebuild_uri(cpv, url) return url def to_ebuild_uri(cpv, url): cat, pkg, ver, rev = portage.catpkgsplit(cpv) - p = '%s-%s' % (pkg, ver) - pvr = '%s%s' % (ver, '-%s' % rev if rev != 'r0' else '') - pf = '%s-%s' % (pkg, pvr) + p = "%s-%s" % (pkg, ver) + pvr = "%s%s" % (ver, "-%s" % rev if rev != "r0" else "") + pf = "%s-%s" % (pkg, pvr) evars = ( - (p, 'P'), - (pkg, 'PN'), - (ver, 'PV'), - (rev, 'PR'), - (pvr, 'PVR'), - (pf, 'PF'), - (cat, 'CATEGORY') + (p, "P"), + (pkg, "PN"), + (ver, "PV"), + (rev, "PR"), + (pvr, "PVR"), + (pf, "PF"), + (cat, "CATEGORY"), ) for src, dst in evars: - url = url.replace(src, '${%s}' % dst) + url = url.replace(src, "${%s}" % dst) return url @@ -112,14 +114,14 @@ def load_mirrors(): def from_mirror(url): - if not url.startswith('mirror://'): + if not url.startswith("mirror://"): return url if not mirrors_: load_mirrors() for mirror_name in mirrors_: - prefix = 'mirror://' + mirror_name + prefix = "mirror://" + mirror_name if url.startswith(prefix): return url.replace(prefix, mirrors_[mirror_name][0]) @@ -137,7 +139,7 @@ def to_mirror(url): return "mirror://%s%s%s" % ( mirror_name, "" if url_part.startswith("/") else "/", - url_part + url_part, ) return url @@ -146,6 +148,7 @@ class EOutputMem(EOutput): """ Override of EOutput, allows to specify an output file for writes """ + def __init__(self, *args, **kwargs): super(EOutputMem, self).__init__(*args, **kwargs) self.out = StringIO() @@ -161,6 +164,7 @@ class EuscanOutput(object): """ Class that handles output for euscan """ + def __init__(self, config): self.config = config self.queries = defaultdict(dict) @@ -212,12 +216,10 @@ class EuscanOutput(object): def result(self, cp, version, urls, handler, confidence): from euscan.version import get_version_type - cpv = '%s-%s' % (cp, version) - urls = ' '.join( - transform_url(self.config, cpv, url) for url in urls.split() - ) + cpv = "%s-%s" % (cp, version) + urls = " ".join(transform_url(self.config, cpv, url) for url in urls.split()) - if self.config['format'] in ['json', 'dict']: + if self.config["format"] in ["json", "dict"]: _curr = self.queries[self.current_query] _curr["result"].append( { @@ -225,12 +227,12 @@ class EuscanOutput(object): "urls": urls.split(), "handler": handler, "confidence": confidence, - "type": get_version_type(version) + "type": get_version_type(version), } ) else: - if not self.config['quiet']: - print("Upstream Version:", pp.number("%s" % version), end=' ') + if not self.config["quiet"]: + print("Upstream Version:", pp.number("%s" % version), end=" ") print(pp.path(" %s" % urls)) else: print(pp.cpv("%s-%s" % (cp, version)) + ":", pp.path(urls)) diff --git a/src/euscan/scan.py b/src/euscan/scan.py index d3c47d4..f16cade 100644 --- a/src/euscan/scan.py +++ b/src/euscan/scan.py @@ -1,5 +1,3 @@ - - import os import sys from datetime import datetime @@ -22,7 +20,6 @@ def filter_versions(cp, versions): filtered = {} for url, version, handler, confidence in versions: - # Try to keep the most specific urls (determinted by the length) if version in filtered and len(url) < len(filtered[version]): continue @@ -31,15 +28,16 @@ def filter_versions(cp, versions): if version_blacklisted(cp, version): continue - filtered[version] = { - "url": url, - "handler": handler, - "confidence": confidence - } + filtered[version] = {"url": url, "handler": handler, "confidence": confidence} return [ - (cp, filtered[version]["url"], version, filtered[version]["handler"], - filtered[version]["confidence"]) + ( + cp, + filtered[version]["url"], + version, + filtered[version]["handler"], + filtered[version]["confidence"], + ) for version in filtered ] @@ -52,7 +50,7 @@ def parse_src_uri(uris): while uris: uri = uris.pop() - if '://' not in uri: + if "://" not in uri: continue if uris and uris[-1] == "->": @@ -73,16 +71,16 @@ def reload_gentoolkit(): import gentoolkit # Not used in recent versions - if not hasattr(gentoolkit.package, 'PORTDB'): + if not hasattr(gentoolkit.package, "PORTDB"): return PORTDB = portage.db[portage.root]["porttree"].dbapi - if hasattr(gentoolkit.dbapi, 'PORTDB'): + if hasattr(gentoolkit.dbapi, "PORTDB"): gentoolkit.dbapi.PORTDB = PORTDB - if hasattr(gentoolkit.package, 'PORTDB'): + if hasattr(gentoolkit.package, "PORTDB"): gentoolkit.package.PORTDB = PORTDB - if hasattr(gentoolkit.query, 'PORTDB'): + if hasattr(gentoolkit.query, "PORTDB"): gentoolkit.query.PORTDB = PORTDB @@ -104,21 +102,18 @@ def scan_upstream(query, on_progress=None): ) if not matches: - output.ewarn( - pp.warn("No package matching '%s'" % pp.pkgquery(query)) - ) + output.ewarn(pp.warn("No package matching '%s'" % pp.pkgquery(query))) return None matches = sorted(matches) pkg = matches.pop() - while '9999' in pkg.version and len(matches): + while "9999" in pkg.version and len(matches): pkg = matches.pop() if not pkg: output.ewarn( - pp.warn("Package '%s' only have a dev version (9999)" - % pp.pkgquery(pkg.cp)) + pp.warn("Package '%s' only have a dev version (9999)" % pp.pkgquery(pkg.cp)) ) return None @@ -132,42 +127,34 @@ def scan_upstream(query, on_progress=None): on_progress(increment=10) if pkg.cp in BLACKLIST_PACKAGES: - output.ewarn( - pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp)) - ) + output.ewarn(pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp))) return None - if not CONFIG['quiet']: - if not CONFIG['format']: - pp.uprint( - " * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name())) - ) + if not CONFIG["quiet"]: + if not CONFIG["format"]: + pp.uprint(" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name()))) pp.uprint() else: output.metadata("overlay", pp.section(pkg.repo_name())) ebuild_path = pkg.ebuild_path() if ebuild_path: - output.metadata( - "ebuild", pp.path(os.path.normpath(ebuild_path)) - ) + output.metadata("ebuild", pp.path(os.path.normpath(ebuild_path))) uris, homepage, description = pkg.environment( - ('SRC_URI', 'HOMEPAGE', 'DESCRIPTION') + ("SRC_URI", "HOMEPAGE", "DESCRIPTION") ) output.metadata("repository", pkg.repo_name()) output.metadata("homepage", homepage) output.metadata("description", description) else: - uris = pkg.environment('SRC_URI') + uris = pkg.environment("SRC_URI") cpv = pkg.cpv uris = parse_src_uri(uris) - uris_expanded = [ - from_mirror(uri) if 'mirror://' in uri else uri for uri in uris - ] + uris_expanded = [from_mirror(uri) if "mirror://" in uri else uri for uri in uris] pkg._uris = uris pkg._uris_expanded = uris_expanded @@ -187,17 +174,16 @@ def scan_upstream(query, on_progress=None): is_current_version_stable = is_version_stable(ver) if len(result) > 0: - if not (CONFIG['format'] or CONFIG['quiet']): + if not (CONFIG["format"] or CONFIG["quiet"]): print("") for cp, url, version, handler, confidence in result: if CONFIG["ignore-pre-release"]: if not is_version_stable(version): continue if CONFIG["ignore-pre-release-if-stable"]: - if is_current_version_stable and \ - not is_version_stable(version): + if is_current_version_stable and not is_version_stable(version): continue - if CONFIG['progress']: + if CONFIG["progress"]: print("", file=sys.stderr) output.result(cp, version, url, handler, confidence) diff --git a/src/euscan/version.py b/src/euscan/version.py index ebb1829..8ea5b78 100644 --- a/src/euscan/version.py +++ b/src/euscan/version.py @@ -25,24 +25,24 @@ def get_version_type(version): return types[0] # TODO: consider returning all types return "release" + # Stolen from pkg_resources, but importing it is not a good idea -component_re = re.compile(r'(\d+ | [a-z]+ | \.| -)', re.VERBOSE) -replace = \ - {'pre': 'c', 'preview': 'c', '-': 'final-', 'rc': 'c', 'dev': '@'}.get +component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE) +replace = {"pre": "c", "preview": "c", "-": "final-", "rc": "c", "dev": "@"}.get def _parse_version_parts(s): for part in component_re.split(s): part = replace(part, part) - if not part or part == '.': + if not part or part == ".": continue - if part[:1] in '0123456789': - yield part.zfill(8) # pad for numeric comparison + if part[:1] in "0123456789": + yield part.zfill(8) # pad for numeric comparison else: - yield '*' + part + yield "*" + part - yield '*final' # ensure that alpha/beta/candidate are before final + yield "*final" # ensure that alpha/beta/candidate are before final def parse_version(s): @@ -78,12 +78,12 @@ def parse_version(s): """ parts = [] for part in _parse_version_parts(s.lower()): - if part.startswith('*'): - if part < '*final': # remove '-' before a prerelease tag - while parts and parts[-1] == '*final-': + if part.startswith("*"): + if part < "*final": # remove '-' before a prerelease tag + while parts and parts[-1] == "*final-": parts.pop() # remove trailing zeros from each series of numeric parts - while parts and parts[-1] == '00000000': + while parts and parts[-1] == "00000000": parts.pop() parts.append(part) return tuple(parts)