From 12bf1fc75b44857f83f760ce613567ec7486efbd Mon Sep 17 00:00:00 2001 From: volpino Date: Thu, 26 Jul 2012 10:44:10 +0200 Subject: [PATCH] euscan: Redesigning the handlers layout Signed-off-by: volpino --- bin/euscan | 4 +- pym/euscan/handlers/__init__.py | 103 +++++++++++++++------ pym/euscan/handlers/package/__init__.py | 19 ++++ pym/euscan/handlers/{ => package}/watch.py | 14 +-- pym/euscan/handlers/url/__init__.py | 19 ++++ pym/euscan/handlers/{ => url}/cpan.py | 4 - pym/euscan/handlers/{ => url}/generic.py | 55 ++++++----- pym/euscan/handlers/{ => url}/github.py | 4 - pym/euscan/handlers/{ => url}/kde.py | 15 ++- pym/euscan/handlers/{ => url}/php.py | 4 - pym/euscan/handlers/{ => url}/pypi.py | 4 - pym/euscan/handlers/{ => url}/rubygem.py | 4 - pym/euscan/scan.py | 72 ++------------ 13 files changed, 168 insertions(+), 153 deletions(-) create mode 100644 pym/euscan/handlers/package/__init__.py rename pym/euscan/handlers/{ => package}/watch.py (94%) create mode 100644 pym/euscan/handlers/url/__init__.py rename pym/euscan/handlers/{ => url}/cpan.py (98%) rename pym/euscan/handlers/{ => url}/generic.py (80%) rename pym/euscan/handlers/{ => url}/github.py (97%) rename pym/euscan/handlers/{ => url}/kde.py (65%) rename pym/euscan/handlers/{ => url}/php.py (97%) rename pym/euscan/handlers/{ => url}/pypi.py (96%) rename pym/euscan/handlers/{ => url}/rubygem.py (97%) diff --git a/bin/euscan b/bin/euscan index 4469f4e..e0371b4 100755 --- a/bin/euscan +++ b/bin/euscan @@ -130,9 +130,9 @@ def print_usage(_error=None, help=None): " - Ignore non-stable versions only if current\n" + " " * 38 + "version is stable", file=out) print(yellow(" --mirror") + - " - use mirror:// URIs", file=out) + " - use mirror:// URIs", file=out) print(yellow(" --ebuild-uri") + - " - use ebuild variables in URIs", file=out) + " - use ebuild variables in URIs", file=out) print(file=out) if _error in ('packages',) or help: diff --git a/pym/euscan/handlers/__init__.py b/pym/euscan/handlers/__init__.py index 495ef7c..899ef24 100644 --- a/pym/euscan/handlers/__init__.py +++ b/pym/euscan/handlers/__init__.py @@ -1,35 +1,86 @@ -import pkgutil - -# autoimport all modules in this directory and append them to handlers list -handlers = [] -for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): - module = loader.find_module(module_name).load_module(module_name) - handlers.append(module) - -# sort handlers by priority (e.g.: generic should be run lastly) -handlers = sorted( - handlers, - key=lambda handler: handler.PRIORITY, - reverse=True -) +import sys +from euscan import CONFIG, output +from euscan.handlers.package import handlers as pkg_handlers +from euscan.handlers.url import handlers as url_handlers -def find_best_handler(pkg, url): - for handler in handlers: +def find_best_pkg_handler(pkg): + """ + Find the best handler for the given package + """ + for handler in pkg_handlers: + if handler.can_handle(pkg): + return handler + return None + + +def find_best_url_handler(pkg, url): + """ + Find the best handler for the given url + """ + for handler in url_handlers: if handler.can_handle(pkg, url): return handler return None -def scan(pkg, url): - handler = find_best_handler(pkg, url) - if handler: - return handler.scan(pkg, url) - return [] +def scan(pkg, urls, on_progress=None): + """ + Scans upstream for the given package. + First tries if a package wide handler is available, then fallbacks + in url handling. + """ + pkg_handler = find_best_pkg_handler(pkg) + if pkg_handler: + if on_progress: + on_progress(increment=35) + if not CONFIG['quiet'] and not CONFIG['format']: + sys.stdout.write("\n") -def brute_force(pkg, url): - handler = find_best_handler(pkg, url) - if handler: - return handler.brute_force(pkg, url) - return [] + versions = pkg_handler.scan(pkg) + + if on_progress: + on_progress(increment=35) + return versions + + if on_progress: + progress_available = 70 + num_urls = sum([len(urls[fn]) for fn in urls]) + if num_urls > 0: + progress_increment = progress_available / num_urls + else: + progress_increment = 0 + + versions = [] + + for filename in urls: + for url in urls[filename]: + if on_progress and progress_available > 0: + on_progress(increment=progress_increment) + progress_available -= progress_increment + + if not CONFIG['quiet'] and not CONFIG['format']: + sys.stdout.write("\n") + output.einfo("SRC_URI is '%s'" % url) + + if '://' not in url: + output.einfo("Invalid url '%s'" % url) + continue + + try: + url_handler = find_best_url_handler(pkg, url) + versions.extend(url_handler.scan(pkg, url)) + except Exception as e: + output.ewarn( + "Handler failed: [%s] %s" % + (e.__class__.__name__, e.message) + ) + + if versions and CONFIG['oneshot']: + break + + if on_progress and progress_available > 0: + on_progress(increment=progress_available) + + return versions diff --git a/pym/euscan/handlers/package/__init__.py b/pym/euscan/handlers/package/__init__.py new file mode 100644 index 0000000..8530b10 --- /dev/null +++ b/pym/euscan/handlers/package/__init__.py @@ -0,0 +1,19 @@ +""" +Package wide handlers for scanning upstream +""" + +import pkgutil + +handlers = [] + +# autoimport all modules in this directory and append them to handlers list +for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): + module = loader.find_module(module_name).load_module(module_name) + handlers.append(module) + +# sort handlers by priority +handlers = sorted( + handlers, + key=lambda handler: handler.PRIORITY, + reverse=True +) diff --git a/pym/euscan/handlers/watch.py b/pym/euscan/handlers/package/watch.py similarity index 94% rename from pym/euscan/handlers/watch.py rename to pym/euscan/handlers/package/watch.py index a129281..14f25d2 100644 --- a/pym/euscan/handlers/watch.py +++ b/pym/euscan/handlers/package/watch.py @@ -3,7 +3,7 @@ import urllib2 import portage -from euscan.handlers import generic +from euscan.handlers.url import generic from euscan import output, helpers PRIORITY = 100 @@ -15,7 +15,7 @@ CONFIDENCE = 100.0 is_pattern = r"\([^\/]+\)" -def can_handle(pkg, url): +def can_handle(pkg): try: return pkg.metadata._xml_tree.find("upstream").find("watch") \ is not None @@ -116,7 +116,7 @@ def handle_directory_patterns(base, file_pattern): for _, path in scan_data] -def scan(pkg, url): +def scan(pkg): output.einfo("Using watch data") cp, ver, rev = portage.pkgsplit(pkg.cpv) @@ -126,18 +126,14 @@ def scan(pkg, url): if not re.search(is_pattern, base): steps = [(base, file_pattern)] res = generic.scan_directory_recursive( - cp, ver, rev, "", steps, url + cp, ver, rev, "", steps, base ) else: res = [] for step in handle_directory_patterns(base, file_pattern): res += generic.scan_directory_recursive( - cp, ver, rev, "", [step], url + cp, ver, rev, "", [step], base ) results += clean_results(res, versionmangle, urlmangle) return results - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/handlers/url/__init__.py b/pym/euscan/handlers/url/__init__.py new file mode 100644 index 0000000..7328644 --- /dev/null +++ b/pym/euscan/handlers/url/__init__.py @@ -0,0 +1,19 @@ +""" +Url wide handlers for scanning upstream +""" + +import pkgutil + +handlers = [] + +# autoimport all modules in this directory and append them to handlers list +for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): + module = loader.find_module(module_name).load_module(module_name) + handlers.append(module) + +# sort handlers by priority +handlers = sorted( + handlers, + key=lambda handler: handler.PRIORITY, + reverse=True +) diff --git a/pym/euscan/handlers/cpan.py b/pym/euscan/handlers/url/cpan.py similarity index 98% rename from pym/euscan/handlers/cpan.py rename to pym/euscan/handlers/url/cpan.py index a54641f..0f587e1 100644 --- a/pym/euscan/handlers/cpan.py +++ b/pym/euscan/handlers/url/cpan.py @@ -131,7 +131,3 @@ def scan(pkg, url): ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/url/generic.py similarity index 80% rename from pym/euscan/handlers/generic.py rename to pym/euscan/handlers/url/generic.py index 12c4a99..38cfa10 100644 --- a/pym/euscan/handlers/generic.py +++ b/pym/euscan/handlers/url/generic.py @@ -108,43 +108,50 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): def scan(pkg, url): - for bu in SCANDIR_BLACKLIST_URLS: - if re.match(bu, url): - output.einfo("%s is blacklisted by rule %s" % (url, bu)) + if CONFIG["scan-dir"]: + for bu in SCANDIR_BLACKLIST_URLS: + if re.match(bu, url): + output.einfo("%s is blacklisted by rule %s" % (url, bu)) + return [] + + resolved_url = helpers.parse_mirror(url) + if not resolved_url: return [] - resolved_url = helpers.parse_mirror(url) - if not resolved_url: - return [] + cp, ver, rev = portage.pkgsplit(pkg.cpv) - cp, ver, rev = portage.pkgsplit(pkg.cpv) + # 'Hack' for _beta/_rc versions where _ is used instead of - + if ver not in resolved_url: + newver = helpers.version_change_end_sep(ver) + if newver and newver in resolved_url: + output.einfo( + "Version: using %s instead of %s" % (newver, ver) + ) + ver = newver - # 'Hack' for _beta/_rc versions where _ is used instead of - - if ver not in resolved_url: - newver = helpers.version_change_end_sep(ver) - if newver and newver in resolved_url: + template = helpers.template_from_url(resolved_url, ver) + if '${' not in template: output.einfo( - "Version: using %s instead of %s" % (newver, ver) + "Url doesn't seems to depend on version: %s not found in %s" % + (ver, resolved_url) ) - ver = newver + return [] + else: + output.einfo("Scanning: %s" % template) - template = helpers.template_from_url(resolved_url, ver) - if '${' not in template: - output.einfo( - "Url doesn't seems to depend on version: %s not found in %s" % - (ver, resolved_url) - ) - return [] - else: - output.einfo("Scanning: %s" % template) + steps = helpers.generate_scan_paths(template) + ret = scan_directory_recursive(cp, ver, rev, "", steps, url) - steps = helpers.generate_scan_paths(template) - ret = scan_directory_recursive(cp, ver, rev, "", steps, url) + if not ret: + brute_force(pkg, url) return ret def brute_force(pkg, url): + if CONFIG["brute-force"] == 0: + return [] + cp, ver, rev = portage.pkgsplit(pkg.cpv) url = helpers.parse_mirror(url) diff --git a/pym/euscan/handlers/github.py b/pym/euscan/handlers/url/github.py similarity index 97% rename from pym/euscan/handlers/github.py rename to pym/euscan/handlers/url/github.py index 9bb5596..080a559 100644 --- a/pym/euscan/handlers/github.py +++ b/pym/euscan/handlers/url/github.py @@ -52,7 +52,3 @@ def scan(pkg, url): if helpers.version_filtered(cp, ver, pv): continue yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE) - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/handlers/kde.py b/pym/euscan/handlers/url/kde.py similarity index 65% rename from pym/euscan/handlers/kde.py rename to pym/euscan/handlers/url/kde.py index 21722bb..5535158 100644 --- a/pym/euscan/handlers/kde.py +++ b/pym/euscan/handlers/url/kde.py @@ -1,4 +1,4 @@ -from euscan.handlers import generic +from euscan.handlers.url import generic PRIORITY = 90 @@ -29,14 +29,11 @@ def scan(pkg, url): url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/') results += generic.scan(pkg.cpv, url) - return clean_results(results) + if not results: # if nothing was found go brute forcing + results = generic.brute_force(pkg.cpv, url) - -def brute_force(pkg, url): - results = generic.brute_force(pkg.cpv, url) - - if url.startswith('mirror://kde/unstable/'): - url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/') - results += generic.brute_force(pkg.cpv, url) + if url.startswith('mirror://kde/unstable/'): + url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/') + results += generic.brute_force(pkg.cpv, url) return clean_results(results) diff --git a/pym/euscan/handlers/php.py b/pym/euscan/handlers/url/php.py similarity index 97% rename from pym/euscan/handlers/php.py rename to pym/euscan/handlers/url/php.py index 6b74ff6..853059a 100644 --- a/pym/euscan/handlers/php.py +++ b/pym/euscan/handlers/url/php.py @@ -70,7 +70,3 @@ def scan(pkg, url): ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/handlers/pypi.py b/pym/euscan/handlers/url/pypi.py similarity index 96% rename from pym/euscan/handlers/pypi.py rename to pym/euscan/handlers/url/pypi.py index 9cd1620..8ed8021 100644 --- a/pym/euscan/handlers/pypi.py +++ b/pym/euscan/handlers/url/pypi.py @@ -52,7 +52,3 @@ def scan(pkg, url): ret.append((urls, pv, HANDLER_NAME, CONFIDENCE)) return ret - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/handlers/rubygem.py b/pym/euscan/handlers/url/rubygem.py similarity index 97% rename from pym/euscan/handlers/rubygem.py rename to pym/euscan/handlers/url/rubygem.py index 39e2334..950e81b 100644 --- a/pym/euscan/handlers/rubygem.py +++ b/pym/euscan/handlers/url/rubygem.py @@ -71,7 +71,3 @@ def scan(pkg, url): ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py index 114c81e..5d9fd06 100644 --- a/pym/euscan/scan.py +++ b/pym/euscan/scan.py @@ -44,60 +44,6 @@ def filter_versions(cp, versions): ] -def scan_upstream_urls(pkg, urls, on_progress): - versions = [] - - if on_progress: - progress_available = 70 - num_urls = sum([len(urls[fn]) for fn in urls]) - if num_urls > 0: - progress_increment = progress_available / num_urls - else: - progress_increment = 0 - - for filename in urls: - for url in urls[filename]: - - if on_progress and progress_available > 0: - on_progress(increment=progress_increment) - progress_available -= progress_increment - - if not CONFIG['quiet'] and not CONFIG['format']: - pp.uprint() - output.einfo("SRC_URI is '%s'" % url) - - if '://' not in url: - output.einfo("Invalid url '%s'" % url) - continue - - # Try normal scan - if CONFIG["scan-dir"]: - try: - versions.extend(handlers.scan(pkg, url)) - except Exception as e: - output.ewarn("Handler failed: [%s] %s" - % (e.__class__.__name__, e.message)) - - if versions and CONFIG['oneshot']: - break - - # Brute Force - if CONFIG["brute-force"] > 0: - versions.extend(handlers.brute_force(pkg, url)) - - if versions and CONFIG['oneshot']: - break - - cp, ver, rev = portage.pkgsplit(pkg.cpv) - - result = filter_versions(cp, versions) - - if on_progress and progress_available > 0: - on_progress(increment=progress_available) - - return result - - # gentoolkit stores PORTDB, so even if we modify it to add an overlay # it will still use the old dbapi def reload_gentoolkit(): @@ -120,7 +66,6 @@ def scan_upstream(query, on_progress=None): """ Scans the upstream searching new versions for the given query """ - matches = [] if query.endswith(".ebuild"): @@ -188,10 +133,6 @@ def scan_upstream(query, on_progress=None): output.metadata("description", pkg.environment("DESCRIPTION")) cpv = pkg.cpv - - _, _, ver, _ = portage.catpkgsplit(cpv) - is_current_version_stable = is_version_stable(ver) - metadata = { "EAPI": portage.settings["EAPI"], "SRC_URI": pkg.environment("SRC_URI", False), @@ -212,15 +153,20 @@ def scan_upstream(query, on_progress=None): else: urls = alist - # output scan time for formatted output - scan_time = (datetime.now() - start_time).total_seconds() - output.metadata("scan_time", scan_time, show=False) + versions = handlers.scan(pkg, urls, on_progress) - result = scan_upstream_urls(pkg, urls, on_progress) + cp, ver, rev = portage.pkgsplit(pkg.cpv) + + result = filter_versions(cp, versions) if on_progress: on_progress(increment=10) + # output scan time for formatted output + scan_time = (datetime.now() - start_time).total_seconds() + output.metadata("scan_time", scan_time, show=False) + + is_current_version_stable = is_version_stable(ver) if len(result) > 0: if not (CONFIG['format'] or CONFIG['quiet']): print("\n", file=sys.stderr)