Merge remote-tracking branch 'fox/master'

Conflicts:
	euscanwww/djeuscan/helpers.py

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>

pym/euscan/handlers/__init__.py

@@ -1,35 +1,86 @@
-import pkgutil
-
-# autoimport all modules in this directory and append them to handlers list
-handlers = []
-for loader, module_name, is_pkg in  pkgutil.walk_packages(__path__):
-    module = loader.find_module(module_name).load_module(module_name)
-    handlers.append(module)
-
-# sort handlers by priority (e.g.: generic should be run lastly)
-handlers = sorted(
-    handlers,
-    key=lambda handler: handler.PRIORITY,
-    reverse=True
-)
+import sys
+from euscan import CONFIG, output
+from euscan.handlers.package import handlers as pkg_handlers
+from euscan.handlers.url import handlers as url_handlers
 
 
-def find_best_handler(pkg, url):
-    for handler in handlers:
+def find_best_pkg_handler(pkg):
+    """
+    Find the best handler for the given package
+    """
+    for handler in pkg_handlers:
+        if handler.can_handle(pkg):
+            return handler
+    return None
+
+
+def find_best_url_handler(pkg, url):
+    """
+    Find the best handler for the given url
+    """
+    for handler in url_handlers:
         if handler.can_handle(pkg, url):
             return handler
     return None
 
 
-def scan(pkg, url):
-    handler = find_best_handler(pkg, url)
-    if handler:
-        return handler.scan(pkg, url)
-    return []
+def scan(pkg, urls, on_progress=None):
+    """
+    Scans upstream for the given package.
+    First tries if a package wide handler is available, then fallbacks
+    in url handling.
+    """
+    pkg_handler = find_best_pkg_handler(pkg)
+    if pkg_handler:
+        if on_progress:
+            on_progress(increment=35)
 
+        if not CONFIG['quiet'] and not CONFIG['format']:
+            sys.stdout.write("\n")
 
-def brute_force(pkg, url):
-    handler = find_best_handler(pkg, url)
-    if handler:
-        return handler.brute_force(pkg, url)
-    return []
+        versions = pkg_handler.scan(pkg)
+
+        if on_progress:
+            on_progress(increment=35)
+        return versions
+
+    if on_progress:
+        progress_available = 70
+        num_urls = sum([len(urls[fn]) for fn in urls])
+        if num_urls > 0:
+            progress_increment = progress_available / num_urls
+        else:
+            progress_increment = 0
+
+    versions = []
+
+    for filename in urls:
+        for url in urls[filename]:
+            if on_progress and progress_available > 0:
+                on_progress(increment=progress_increment)
+                progress_available -= progress_increment
+
+            if not CONFIG['quiet'] and not CONFIG['format']:
+                sys.stdout.write("\n")
+            output.einfo("SRC_URI is '%s'" % url)
+
+            if '://' not in url:
+                output.einfo("Invalid url '%s'" % url)
+                continue
+
+            try:
+                url_handler = find_best_url_handler(pkg, url)
+                versions.extend(url_handler.scan(pkg, url))
+            except Exception as e:
+                output.ewarn(
+                    "Handler failed: [%s] %s" %
+                    (e.__class__.__name__, e.message)
+                )
+
+            if versions and CONFIG['oneshot']:
+                break
+
+    if on_progress and progress_available > 0:
+        on_progress(increment=progress_available)
+
+    return versions

pym/euscan/handlers/package/__init__.py

@@ -0,0 +1,19 @@
+"""
+Package wide handlers for scanning upstream
+"""
+
+import pkgutil
+
+handlers = []
+
+# autoimport all modules in this directory and append them to handlers list
+for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
+    module = loader.find_module(module_name).load_module(module_name)
+    handlers.append(module)
+
+# sort handlers by priority
+handlers = sorted(
+    handlers,
+    key=lambda handler: handler.PRIORITY,
+    reverse=True
+)

pym/euscan/handlers/package/remote_id.py

@@ -0,0 +1,43 @@
+from euscan.handlers.url import handlers
+from euscan import output
+
+PRIORITY = 100
+
+HANDLER_NAME = "remote_id"
+CONFIDENCE = 100.0
+
+
+url_handlers = {handler.HANDLER_NAME: handler for handler in handlers}
+
+
+def can_handle(pkg):
+    # Return True if there's at least one remote-id that can be
+    # handled by euscan
+    try:
+        remoteids = pkg.metadata.upstream()[0].upstream_remoteids()
+    except IndexError:
+        pass
+    else:
+        if len(remoteids) > 0:
+            for remote_value, remote_type in remoteids:
+                if remote_type in url_handlers:
+                    return True
+    return False
+
+
+def scan(pkg):
+    output.einfo("Using remote-id data")
+
+    ret = []
+
+    remoteids = pkg.metadata.upstream()[0].upstream_remoteids()
+    for remote_value, remote_type in remoteids:
+        if remote_type in url_handlers:
+            remote_data = remote_value.split("/")
+            scan_remote = getattr(
+                url_handlers[remote_type], "scan_remote", None
+            )
+            if scan_remote:
+                for url, pv in scan_remote(pkg, remote_data):
+                    ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
+    return ret

pym/euscan/handlers/package/watch.py

@@ -3,7 +3,7 @@ import urllib2
 
 import portage
 
-from euscan.handlers import generic
+from euscan.handlers.url import generic
 from euscan import output, helpers
 
 PRIORITY = 100
@@ -15,7 +15,7 @@ CONFIDENCE = 100.0
 is_pattern = r"\([^\/]+\)"
 
 
-def can_handle(pkg, url):
+def can_handle(pkg):
     try:
         return pkg.metadata._xml_tree.find("upstream").find("watch") \
                is not None
@@ -116,7 +116,7 @@ def handle_directory_patterns(base, file_pattern):
             for _, path in scan_data]
 
 
-def scan(pkg, url):
+def scan(pkg):
     output.einfo("Using watch data")
 
     cp, ver, rev = portage.pkgsplit(pkg.cpv)
@@ -126,18 +126,14 @@ def scan(pkg, url):
         if not re.search(is_pattern, base):
             steps = [(base, file_pattern)]
             res = generic.scan_directory_recursive(
-                cp, ver, rev, "", steps, url
+                cp, ver, rev, "", steps, base
             )
         else:
             res = []
             for step in handle_directory_patterns(base, file_pattern):
                 res += generic.scan_directory_recursive(
-                    cp, ver, rev, "", [step], url
+                    cp, ver, rev, "", [step], base
                 )
 
         results += clean_results(res, versionmangle, urlmangle)
     return results
-
-
-def brute_force(pkg, url):
-    return []

pym/euscan/handlers/url/__init__.py

@@ -0,0 +1,19 @@
+"""
+Url wide handlers for scanning upstream
+"""
+
+import pkgutil
+
+handlers = []
+
+# autoimport all modules in this directory and append them to handlers list
+for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
+    module = loader.find_module(module_name).load_module(module_name)
+    handlers.append(module)
+
+# sort handlers by priority
+handlers = sorted(
+    handlers,
+    key=lambda handler: handler.PRIORITY,
+    reverse=True
+)

pym/euscan/handlers/url/cpan.py

@@ -81,12 +81,22 @@ def cpan_vercmp(cp, a, b):
 
 def scan(pkg, url):
     cp, ver, rev = portage.pkgsplit(pkg.cpv)
-    pkg = guess_package(cp, url)
+    remote_pkg = guess_package(cp, url)
 
-    orig_url = url
-    url = 'http://search.cpan.org/api/dist/%s' % pkg
+    output.einfo("Using CPAN API: %s", remote_pkg)
 
-    output.einfo("Using: " + url)
+    result = scan_remote(pkg, [remote_pkg])
+
+    ret = []
+    for url, pv in result:
+        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
+    return ret
+
+
+def scan_remote(pkg, remote_data):
+    remote_pkg = remote_data[0]
+    url = 'http://search.cpan.org/api/dist/%s' % remote_pkg
+    cp, ver, rev = portage.pkgsplit(pkg.cpv)
 
     try:
         fp = helpers.urlopen(url)
@@ -125,13 +135,6 @@ def scan(pkg, url):
             version['archive']
         )
 
-        if url == orig_url:
-            continue
-
-        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
+        ret.append((url, pv))
 
     return ret
-
-
-def brute_force(pkg, url):
-    return []

pym/euscan/handlers/url/generic.py

@@ -108,43 +108,50 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
 
 
 def scan(pkg, url):
-    for bu in SCANDIR_BLACKLIST_URLS:
-        if re.match(bu, url):
-            output.einfo("%s is blacklisted by rule %s" % (url, bu))
+    if CONFIG["scan-dir"]:
+        for bu in SCANDIR_BLACKLIST_URLS:
+            if re.match(bu, url):
+                output.einfo("%s is blacklisted by rule %s" % (url, bu))
+                return []
+
+        resolved_url = helpers.parse_mirror(url)
+        if not resolved_url:
             return []
 
-    resolved_url = helpers.parse_mirror(url)
-    if not resolved_url:
-        return []
+        cp, ver, rev = portage.pkgsplit(pkg.cpv)
 
-    cp, ver, rev = portage.pkgsplit(pkg.cpv)
+        # 'Hack' for _beta/_rc versions where _ is used instead of -
+        if ver not in resolved_url:
+            newver = helpers.version_change_end_sep(ver)
+            if newver and newver in resolved_url:
+                output.einfo(
+                    "Version: using %s instead of %s" % (newver, ver)
+                )
+                ver = newver
 
-    # 'Hack' for _beta/_rc versions where _ is used instead of -
-    if ver not in resolved_url:
-        newver = helpers.version_change_end_sep(ver)
-        if newver and newver in resolved_url:
+        template = helpers.template_from_url(resolved_url, ver)
+        if '${' not in template:
             output.einfo(
-                "Version: using %s instead of %s" % (newver, ver)
+                "Url doesn't seems to depend on version: %s not found in %s" %
+                (ver, resolved_url)
             )
-            ver = newver
+            return []
+        else:
+            output.einfo("Scanning: %s" % template)
 
-    template = helpers.template_from_url(resolved_url, ver)
-    if '${' not in template:
-        output.einfo(
-            "Url doesn't seems to depend on version: %s not found in %s" %
-            (ver, resolved_url)
-        )
-        return []
-    else:
-        output.einfo("Scanning: %s" % template)
+        steps = helpers.generate_scan_paths(template)
+        ret = scan_directory_recursive(cp, ver, rev, "", steps, url)
 
-    steps = helpers.generate_scan_paths(template)
-    ret = scan_directory_recursive(cp, ver, rev, "", steps, url)
+    if not ret:
+        ret = brute_force(pkg, url)
 
     return ret
 
 
 def brute_force(pkg, url):
+    if CONFIG["brute-force"] == 0:
+        return []
+
     cp, ver, rev = portage.pkgsplit(pkg.cpv)
 
     url = helpers.parse_mirror(url)

pym/euscan/handlers/url/github.py

@@ -44,6 +44,7 @@ def scan(pkg, url):
                             (user, project))
     dls = json.load(dlreq)
 
+    ret = []
     for dl in dls:
         m = fnre.match(dl['name'])
 
@@ -51,8 +52,6 @@ def scan(pkg, url):
             pv = helpers.gentoo_mangle_version(m.group(1))
             if helpers.version_filtered(cp, ver, pv):
                 continue
-            yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE)
 
-
-def brute_force(pkg, url):
-    return []
+            ret.append((dl['html_url'], pv, HANDLER_NAME, CONFIDENCE))
+    return ret

pym/euscan/handlers/url/kde.py

@@ -1,4 +1,4 @@
-from euscan.handlers import generic
+from euscan.handlers.url import generic
 
 PRIORITY = 90
 
@@ -29,14 +29,11 @@ def scan(pkg, url):
         url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/')
         results += generic.scan(pkg.cpv, url)
 
-    return clean_results(results)
+    if not results:  # if nothing was found go brute forcing
+        results = generic.brute_force(pkg.cpv, url)
 
-
-def brute_force(pkg, url):
-    results = generic.brute_force(pkg.cpv, url)
-
-    if url.startswith('mirror://kde/unstable/'):
-        url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/')
-        results += generic.brute_force(pkg.cpv, url)
+        if url.startswith('mirror://kde/unstable/'):
+            url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/')
+            results += generic.brute_force(pkg.cpv, url)
 
     return clean_results(results)

pym/euscan/handlers/url/php.py

@@ -70,7 +70,3 @@ def scan(pkg, url):
         ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
 
     return ret
-
-
-def brute_force(pkg, url):
-    return []

pym/euscan/handlers/url/pypi.py

@@ -29,6 +29,15 @@ def scan(pkg, url):
 
     package = guess_package(pkg.cpv, url)
 
+    ret = []
+    for urls, pv in scan_remote(pkg, [package]):
+        ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
+    return ret
+
+
+def scan_remote(pkg, remote_data):
+    package = remote_data[0]
+
     output.einfo("Using PyPi XMLRPC: " + package)
 
     client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
@@ -42,17 +51,11 @@ def scan(pkg, url):
     cp, ver, rev = portage.pkgsplit(pkg.cpv)
 
     ret = []
-
     for up_pv in versions:
         pv = helpers.gentoo_mangle_version(up_pv)
         if helpers.version_filtered(cp, ver, pv):
             continue
         urls = client.release_urls(package, up_pv)
         urls = " ".join([infos['url'] for infos in urls])
-        ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
-
+        ret.append((urls, pv))
     return ret
-
-
-def brute_force(pkg, url):
-    return []

pym/euscan/handlers/url/rubygems.py

@@ -5,7 +5,7 @@ import urllib2
 
 from euscan import helpers, output
 
-HANDLER_NAME = "rubygem"
+HANDLER_NAME = "rubygems"
 CONFIDENCE = 100.0
 PRIORITY = 90
 
@@ -33,14 +33,23 @@ def scan(pkg, url):
     'http://guides.rubygems.org/rubygems-org-api/#gemversion'
 
     gem = guess_gem(pkg.cpv, url)
+
     if not gem:
         output.eerror("Can't guess gem name using %s and %s" % \
             (pkg.cpv, url))
         return []
 
-    url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
+    output.einfo("Using RubyGem API: %s" % gem)
 
-    output.einfo("Using: " + url)
+    ret = []
+    for url, pv in scan_remote(pkg, [gem]):
+        ret.append(url, pv, HANDLER_NAME, CONFIDENCE)
+    return ret
+
+
+def scan_remote(pkg, remote_data):
+    gem = remote_data[0]
+    url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
 
     try:
         fp = helpers.urlopen(url)
@@ -55,23 +64,14 @@ def scan(pkg, url):
     data = fp.read()
     versions = json.loads(data)
 
-    if not versions:
-        return []
-
     cp, ver, rev = portage.pkgsplit(pkg.cpv)
 
     ret = []
-
     for version in versions:
         up_pv = version['number']
         pv = helpers.gentoo_mangle_version(up_pv)
         if helpers.version_filtered(cp, ver, pv):
             continue
         url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
-        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
-
+        ret.append((url, pv))
     return ret
-
-
-def brute_force(pkg, url):
-    return []

pym/euscan/scan.py

@@ -44,60 +44,6 @@ def filter_versions(cp, versions):
     ]
 
 
-def scan_upstream_urls(pkg, urls, on_progress):
-    versions = []
-
-    if on_progress:
-        progress_available = 70
-        num_urls = sum([len(urls[fn]) for fn in urls])
-        if num_urls > 0:
-            progress_increment = progress_available / num_urls
-        else:
-            progress_increment = 0
-
-    for filename in urls:
-        for url in urls[filename]:
-
-            if on_progress and progress_available > 0:
-                on_progress(increment=progress_increment)
-                progress_available -= progress_increment
-
-            if not CONFIG['quiet'] and not CONFIG['format']:
-                pp.uprint()
-            output.einfo("SRC_URI is '%s'" % url)
-
-            if '://' not in url:
-                output.einfo("Invalid url '%s'" % url)
-                continue
-
-            # Try normal scan
-            if CONFIG["scan-dir"]:
-                try:
-                    versions.extend(handlers.scan(pkg, url))
-                except Exception as e:
-                    output.ewarn("Handler failed: [%s] %s"
-                            % (e.__class__.__name__, e.message))
-
-            if versions and CONFIG['oneshot']:
-                break
-
-            # Brute Force
-            if CONFIG["brute-force"] > 0:
-                versions.extend(handlers.brute_force(pkg, url))
-
-            if versions and CONFIG['oneshot']:
-                break
-
-    cp, ver, rev = portage.pkgsplit(pkg.cpv)
-
-    result = filter_versions(cp, versions)
-
-    if on_progress and progress_available > 0:
-        on_progress(increment=progress_available)
-
-    return result
-
-
 # gentoolkit stores PORTDB, so even if we modify it to add an overlay
 # it will still use the old dbapi
 def reload_gentoolkit():
@@ -120,7 +66,6 @@ def scan_upstream(query, on_progress=None):
     """
     Scans the upstream searching new versions for the given query
     """
-
     matches = []
 
     if query.endswith(".ebuild"):
@@ -188,10 +133,6 @@ def scan_upstream(query, on_progress=None):
         output.metadata("description", pkg.environment("DESCRIPTION"))
 
     cpv = pkg.cpv
-
-    _, _, ver, _ = portage.catpkgsplit(cpv)
-    is_current_version_stable = is_version_stable(ver)
-
     metadata = {
         "EAPI": portage.settings["EAPI"],
         "SRC_URI": pkg.environment("SRC_URI", False),
@@ -212,15 +153,20 @@ def scan_upstream(query, on_progress=None):
     else:
         urls = alist
 
-    # output scan time for formatted output
-    scan_time = (datetime.now() - start_time).total_seconds()
-    output.metadata("scan_time", scan_time, show=False)
+    versions = handlers.scan(pkg, urls, on_progress)
 
-    result = scan_upstream_urls(pkg, urls, on_progress)
+    cp, ver, rev = portage.pkgsplit(pkg.cpv)
+
+    result = filter_versions(cp, versions)
 
     if on_progress:
         on_progress(increment=10)
 
+    # output scan time for formatted output
+    scan_time = (datetime.now() - start_time).total_seconds()
+    output.metadata("scan_time", scan_time, show=False)
+
+    is_current_version_stable = is_version_stable(ver)
     if len(result) > 0:
         if not (CONFIG['format'] or CONFIG['quiet']):
             print("\n", file=sys.stderr)