euscan: Redesigning the handlers layout

Signed-off-by: volpino <fox91@anche.no>

pym/euscan/handlers/__init__.py

@@ -1,35 +1,86 @@
-import pkgutil
+import sys
-
+from euscan import CONFIG, output
-# autoimport all modules in this directory and append them to handlers list
+from euscan.handlers.package import handlers as pkg_handlers
-handlers = []
+from euscan.handlers.url import handlers as url_handlers
-for loader, module_name, is_pkg in  pkgutil.walk_packages(__path__):
-    module = loader.find_module(module_name).load_module(module_name)
-    handlers.append(module)
-
-# sort handlers by priority (e.g.: generic should be run lastly)
-handlers = sorted(
-    handlers,
-    key=lambda handler: handler.PRIORITY,
-    reverse=True
-)
 
 
-def find_best_handler(pkg, url):
+def find_best_pkg_handler(pkg):
-    for handler in handlers:
+    """
+    Find the best handler for the given package
+    """
+    for handler in pkg_handlers:
+        if handler.can_handle(pkg):
+            return handler
+    return None
+
+
+def find_best_url_handler(pkg, url):
+    """
+    Find the best handler for the given url
+    """
+    for handler in url_handlers:
         if handler.can_handle(pkg, url):
             return handler
     return None
 
 
-def scan(pkg, url):
+def scan(pkg, urls, on_progress=None):
-    handler = find_best_handler(pkg, url)
+    """
-    if handler:
+    Scans upstream for the given package.
-        return handler.scan(pkg, url)
+    First tries if a package wide handler is available, then fallbacks
-    return []
+    in url handling.
+    """
+    pkg_handler = find_best_pkg_handler(pkg)
+    if pkg_handler:
+        if on_progress:
+            on_progress(increment=35)
 
+        if not CONFIG['quiet'] and not CONFIG['format']:
+            sys.stdout.write("\n")
 
-def brute_force(pkg, url):
+        versions = pkg_handler.scan(pkg)
-    handler = find_best_handler(pkg, url)
+
-    if handler:
+        if on_progress:
-        return handler.brute_force(pkg, url)
+            on_progress(increment=35)
-    return []
+        return versions
+
+    if on_progress:
+        progress_available = 70
+        num_urls = sum([len(urls[fn]) for fn in urls])
+        if num_urls > 0:
+            progress_increment = progress_available / num_urls
+        else:
+            progress_increment = 0
+
+    versions = []
+
+    for filename in urls:
+        for url in urls[filename]:
+            if on_progress and progress_available > 0:
+                on_progress(increment=progress_increment)
+                progress_available -= progress_increment
+
+            if not CONFIG['quiet'] and not CONFIG['format']:
+                sys.stdout.write("\n")
+            output.einfo("SRC_URI is '%s'" % url)
+
+            if '://' not in url:
+                output.einfo("Invalid url '%s'" % url)
+                continue
+
+            try:
+                url_handler = find_best_url_handler(pkg, url)
+                versions.extend(url_handler.scan(pkg, url))
+            except Exception as e:
+                output.ewarn(
+                    "Handler failed: [%s] %s" %
+                    (e.__class__.__name__, e.message)
+                )
+
+            if versions and CONFIG['oneshot']:
+                break
+
+    if on_progress and progress_available > 0:
+        on_progress(increment=progress_available)
+
+    return versions

pym/euscan/handlers/package/__init__.py

@@ -0,0 +1,19 @@
+"""
+Package wide handlers for scanning upstream
+"""
+
+import pkgutil
+
+handlers = []
+
+# autoimport all modules in this directory and append them to handlers list
+for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
+    module = loader.find_module(module_name).load_module(module_name)
+    handlers.append(module)
+
+# sort handlers by priority
+handlers = sorted(
+    handlers,
+    key=lambda handler: handler.PRIORITY,
+    reverse=True
+)

pym/euscan/handlers/package/watch.py

@@ -3,7 +3,7 @@ import urllib2
 
 import portage
 
-from euscan.handlers import generic
+from euscan.handlers.url import generic
 from euscan import output, helpers
 
 PRIORITY = 100
@@ -15,7 +15,7 @@ CONFIDENCE = 100.0
 is_pattern = r"\([^\/]+\)"
 
 
-def can_handle(pkg, url):
+def can_handle(pkg):
     try:
         return pkg.metadata._xml_tree.find("upstream").find("watch") \
                is not None
@@ -116,7 +116,7 @@ def handle_directory_patterns(base, file_pattern):
             for _, path in scan_data]
 
 
-def scan(pkg, url):
+def scan(pkg):
     output.einfo("Using watch data")
 
     cp, ver, rev = portage.pkgsplit(pkg.cpv)
@@ -126,18 +126,14 @@ def scan(pkg, url):
         if not re.search(is_pattern, base):
             steps = [(base, file_pattern)]
             res = generic.scan_directory_recursive(
-                cp, ver, rev, "", steps, url
+                cp, ver, rev, "", steps, base
             )
         else:
             res = []
             for step in handle_directory_patterns(base, file_pattern):
                 res += generic.scan_directory_recursive(
-                    cp, ver, rev, "", [step], url
+                    cp, ver, rev, "", [step], base
                 )
 
         results += clean_results(res, versionmangle, urlmangle)
     return results
-
-
-def brute_force(pkg, url):
-    return []

pym/euscan/handlers/url/__init__.py

@@ -0,0 +1,19 @@
+"""
+Url wide handlers for scanning upstream
+"""
+
+import pkgutil
+
+handlers = []
+
+# autoimport all modules in this directory and append them to handlers list
+for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
+    module = loader.find_module(module_name).load_module(module_name)
+    handlers.append(module)
+
+# sort handlers by priority
+handlers = sorted(
+    handlers,
+    key=lambda handler: handler.PRIORITY,
+    reverse=True
+)

pym/euscan/handlers/url/cpan.py

@@ -131,7 +131,3 @@ def scan(pkg, url):
         ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
 
     return ret
-
-
-def brute_force(pkg, url):
-    return []

pym/euscan/handlers/url/generic.py

@@ -108,6 +108,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
 
 
 def scan(pkg, url):
+    if CONFIG["scan-dir"]:
         for bu in SCANDIR_BLACKLIST_URLS:
             if re.match(bu, url):
                 output.einfo("%s is blacklisted by rule %s" % (url, bu))
@@ -141,10 +142,16 @@ def scan(pkg, url):
         steps = helpers.generate_scan_paths(template)
         ret = scan_directory_recursive(cp, ver, rev, "", steps, url)
 
+    if not ret:
+        brute_force(pkg, url)
+
     return ret
 
 
 def brute_force(pkg, url):
+    if CONFIG["brute-force"] == 0:
+        return []
+
     cp, ver, rev = portage.pkgsplit(pkg.cpv)
 
     url = helpers.parse_mirror(url)

pym/euscan/handlers/url/github.py

@@ -52,7 +52,3 @@ def scan(pkg, url):
             if helpers.version_filtered(cp, ver, pv):
                 continue
             yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE)
-
-
-def brute_force(pkg, url):
-    return []

pym/euscan/handlers/url/kde.py

@@ -1,4 +1,4 @@
-from euscan.handlers import generic
+from euscan.handlers.url import generic
 
 PRIORITY = 90
 
@@ -29,10 +29,7 @@ def scan(pkg, url):
         url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/')
         results += generic.scan(pkg.cpv, url)
 
-    return clean_results(results)
+    if not results:  # if nothing was found go brute forcing
-
-
-def brute_force(pkg, url):
         results = generic.brute_force(pkg.cpv, url)
 
         if url.startswith('mirror://kde/unstable/'):

pym/euscan/handlers/url/php.py

@@ -70,7 +70,3 @@ def scan(pkg, url):
         ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
 
     return ret
-
-
-def brute_force(pkg, url):
-    return []

pym/euscan/handlers/url/pypi.py

@@ -52,7 +52,3 @@ def scan(pkg, url):
         ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
 
     return ret
-
-
-def brute_force(pkg, url):
-    return []

pym/euscan/handlers/url/rubygem.py

@@ -71,7 +71,3 @@ def scan(pkg, url):
         ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
 
     return ret
-
-
-def brute_force(pkg, url):
-    return []

pym/euscan/scan.py

@@ -44,60 +44,6 @@ def filter_versions(cp, versions):
     ]
 
 
-def scan_upstream_urls(pkg, urls, on_progress):
-    versions = []
-
-    if on_progress:
-        progress_available = 70
-        num_urls = sum([len(urls[fn]) for fn in urls])
-        if num_urls > 0:
-            progress_increment = progress_available / num_urls
-        else:
-            progress_increment = 0
-
-    for filename in urls:
-        for url in urls[filename]:
-
-            if on_progress and progress_available > 0:
-                on_progress(increment=progress_increment)
-                progress_available -= progress_increment
-
-            if not CONFIG['quiet'] and not CONFIG['format']:
-                pp.uprint()
-            output.einfo("SRC_URI is '%s'" % url)
-
-            if '://' not in url:
-                output.einfo("Invalid url '%s'" % url)
-                continue
-
-            # Try normal scan
-            if CONFIG["scan-dir"]:
-                try:
-                    versions.extend(handlers.scan(pkg, url))
-                except Exception as e:
-                    output.ewarn("Handler failed: [%s] %s"
-                            % (e.__class__.__name__, e.message))
-
-            if versions and CONFIG['oneshot']:
-                break
-
-            # Brute Force
-            if CONFIG["brute-force"] > 0:
-                versions.extend(handlers.brute_force(pkg, url))
-
-            if versions and CONFIG['oneshot']:
-                break
-
-    cp, ver, rev = portage.pkgsplit(pkg.cpv)
-
-    result = filter_versions(cp, versions)
-
-    if on_progress and progress_available > 0:
-        on_progress(increment=progress_available)
-
-    return result
-
-
 # gentoolkit stores PORTDB, so even if we modify it to add an overlay
 # it will still use the old dbapi
 def reload_gentoolkit():
@@ -120,7 +66,6 @@ def scan_upstream(query, on_progress=None):
     """
     Scans the upstream searching new versions for the given query
     """
-
     matches = []
 
     if query.endswith(".ebuild"):
@@ -188,10 +133,6 @@ def scan_upstream(query, on_progress=None):
         output.metadata("description", pkg.environment("DESCRIPTION"))
 
     cpv = pkg.cpv
-
-    _, _, ver, _ = portage.catpkgsplit(cpv)
-    is_current_version_stable = is_version_stable(ver)
-
     metadata = {
         "EAPI": portage.settings["EAPI"],
         "SRC_URI": pkg.environment("SRC_URI", False),
@@ -212,15 +153,20 @@ def scan_upstream(query, on_progress=None):
     else:
         urls = alist
 
-    # output scan time for formatted output
+    versions = handlers.scan(pkg, urls, on_progress)
-    scan_time = (datetime.now() - start_time).total_seconds()
-    output.metadata("scan_time", scan_time, show=False)
 
-    result = scan_upstream_urls(pkg, urls, on_progress)
+    cp, ver, rev = portage.pkgsplit(pkg.cpv)
+
+    result = filter_versions(cp, versions)
 
     if on_progress:
         on_progress(increment=10)
 
+    # output scan time for formatted output
+    scan_time = (datetime.now() - start_time).total_seconds()
+    output.metadata("scan_time", scan_time, show=False)
+
+    is_current_version_stable = is_version_stable(ver)
     if len(result) > 0:
         if not (CONFIG['format'] or CONFIG['quiet']):
             print("\n", file=sys.stderr)