euscan: json format output

Now "-f json" seems good, it supports the handler type used to retrieve
each version and outputs metadata.

Signed-off-by: volpino <fox91@anche.no>

pym/euscan/handlers/cpan.py

@@ -3,8 +3,9 @@ import portage
 import urllib2
 import json
 
-from euscan import helpers
-import euscan
+from euscan import helpers, output
+
+HANDLER_NAME = "cpan"
 
 _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*")
 
@@ -83,7 +84,7 @@ def scan(cpv, url):
     orig_url = url
     url = 'http://search.cpan.org/api/dist/%s' % pkg
 
-    euscan.output.einfo("Using: " + url)
+    output.einfo("Using: " + url)
 
     try:
         fp = helpers.urlopen(url)
@@ -125,7 +126,7 @@ def scan(cpv, url):
         if url == orig_url:
             continue
 
-        ret.append((url, pv))
+        ret.append((url, pv, HANDLER_NAME))
 
     return ret
 

pym/euscan/handlers/generic.py

@@ -7,9 +7,10 @@ from BeautifulSoup import BeautifulSoup
 import portage
 
 from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \
-    BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS
-from euscan import helpers
-import euscan
+    BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS, output, helpers
+
+HANDLER_NAME = "generic"
+BRUTEFORCE_HANDLER_NAME = "brute_force"
 
 
 def scan_html(data, url, pattern):
@@ -53,7 +54,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
 
     steps = steps[1:]
 
-    euscan.output.einfo("Scanning: %s" % url)
+    output.einfo("Scanning: %s" % url)
 
     try:
         fp = helpers.urlopen(url)
@@ -87,7 +88,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
             path = url + path
 
         if not steps and path not in orig_url:
-            versions.append((path, pv))
+            versions.append((path, pv, HANDLER_NAME))
 
         if steps:
             ret = scan_directory_recursive(cp, ver, rev, path, steps, orig_url)
@@ -99,7 +100,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
 def scan(cpv, url):
     for bu in SCANDIR_BLACKLIST_URLS:
         if re.match(bu, url):
-            euscan.output.einfo("%s is blacklisted by rule %s" % (url, bu))
+            output.einfo("%s is blacklisted by rule %s" % (url, bu))
             return []
 
     resolved_url = helpers.parse_mirror(url)
@@ -112,23 +113,25 @@ def scan(cpv, url):
     if ver not in resolved_url:
         newver = helpers.version_change_end_sep(ver)
         if newver and newver in resolved_url:
-            euscan.output.einfo(
+            output.einfo(
                 "Version: using %s instead of %s" % (newver, ver)
             )
             ver = newver
 
     template = helpers.template_from_url(resolved_url, ver)
     if '${' not in template:
-        euscan.output.einfo(
+        output.einfo(
             "Url doesn't seems to depend on version: %s not found in %s" %
             (ver, resolved_url)
         )
         return []
     else:
-        euscan.output.einfo("Scanning: %s" % template)
+        output.einfo("Scanning: %s" % template)
 
     steps = helpers.generate_scan_paths(template)
-    return scan_directory_recursive(cp, ver, rev, "", steps, url)
+    ret = scan_directory_recursive(cp, ver, rev, "", steps, url)
+
+    return ret
 
 
 def brute_force(cpv, url):
@@ -140,15 +143,15 @@ def brute_force(cpv, url):
 
     for bp in BRUTEFORCE_BLACKLIST_PACKAGES:
         if re.match(bp, cp):
-            euscan.output.einfo("%s is blacklisted by rule %s" % (cp, bp))
+            output.einfo("%s is blacklisted by rule %s" % (cp, bp))
             return []
 
     for bp in BRUTEFORCE_BLACKLIST_URLS:
         if re.match(bp, url):
-            euscan.output.einfo("%s is blacklisted by rule %s" % (cp, bp))
+            output.einfo("%s is blacklisted by rule %s" % (cp, bp))
             return []
 
-    euscan.output.einfo("Generating version from " + ver)
+    output.einfo("Generating version from " + ver)
 
     components = helpers.split_version(ver)
     versions = helpers.gen_versions(components, CONFIG["brute-force"])
@@ -159,18 +162,18 @@ def brute_force(cpv, url):
             versions.remove(v)
 
     if not versions:
-        euscan.output.einfo("Can't generate new versions from " + ver)
+        output.einfo("Can't generate new versions from " + ver)
         return []
 
     template = helpers.template_from_url(url, ver)
 
     if '${PV}' not in template:
-        euscan.output.einfo(
+        output.einfo(
             "Url doesn't seems to depend on full version: %s not found in %s" %
             (ver, url))
         return []
     else:
-        euscan.output.einfo("Brute forcing: %s" % template)
+        output.einfo("Brute forcing: %s" % template)
 
     result = []
 
@@ -195,10 +198,10 @@ def brute_force(cpv, url):
         if not infos:
             continue
 
-        result.append([url, version])
+        result.append([url, version, BRUTEFORCE_HANDLER_NAME])
 
         if len(result) > CONFIG['brute-force-false-watermark']:
-            euscan.output.einfo(
+            output.einfo(
                 "Broken server detected ! Skipping brute force."
             )
             return []

pym/euscan/handlers/kde.py

@@ -1,5 +1,7 @@
 from euscan.handlers import generic
 
+HANDLER_NAME = "kde"
+
 
 def can_handle(cpv, url):
     if url.startswith('mirror://kde/'):
@@ -13,7 +15,7 @@ def clean_results(results):
     for path, version in results:
         if version == '5SUMS':
             continue
-        ret.append((path, version))
+        ret.append((path, version, HANDLER_NAME))
 
     return ret
 

pym/euscan/handlers/php.py

@@ -3,8 +3,9 @@ import portage
 import urllib2
 import xml.dom.minidom
 
-from euscan import helpers
-import euscan
+from euscan import helpers, output
+
+HANDLER_NAME = "php"
 
 
 def can_handle(cpv, url):
@@ -34,7 +35,7 @@ def scan(cpv, url):
     orig_url = url
     url = 'http://%s/rest/r/%s/allreleases.xml' % (channel, pkg.lower())
 
-    euscan.output.einfo("Using: " + url)
+    output.einfo("Using: " + url)
 
     try:
         fp = helpers.urlopen(url)
@@ -64,7 +65,7 @@ def scan(cpv, url):
         if url == orig_url:
             continue
 
-        ret.append((url, pv))
+        ret.append((url, pv, HANDLER_NAME))
 
     return ret
 

pym/euscan/handlers/pypi.py

@@ -3,8 +3,9 @@ import re
 
 import portage
 
-from euscan import helpers
-import euscan
+from euscan import helpers, output
+
+HANDLER_NAME = "pypi"
 
 
 def can_handle(cpv, url):
@@ -26,7 +27,7 @@ def scan(cpv, url):
 
     package = guess_package(cpv, url)
 
-    euscan.output.einfo("Using PyPi XMLRPC: " + package)
+    output.einfo("Using PyPi XMLRPC: " + package)
 
     client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
     versions = client.package_releases(package)
@@ -46,7 +47,7 @@ def scan(cpv, url):
             continue
         urls = client.release_urls(package, up_pv)
         urls = " ".join([infos['url'] for infos in urls])
-        ret.append((urls, pv))
+        ret.append((urls, pv, HANDLER_NAME))
 
     return ret
 

pym/euscan/handlers/rubygem.py

@@ -3,8 +3,9 @@ import portage
 import json
 import urllib2
 
-from euscan import helpers
-import euscan
+from euscan import helpers, output
+
+HANDLER_NAME = "rubygem"
 
 
 def can_handle(cpv, url):
@@ -31,13 +32,13 @@ def scan(cpv, url):
 
     gem = guess_gem(cpv, url)
     if not gem:
-        euscan.output.eerror("Can't guess gem name using %s and %s" % \
+        output.eerror("Can't guess gem name using %s and %s" % \
             (cpv, url))
         return []
 
     url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
 
-    euscan.output.einfo("Using: " + url)
+    output.einfo("Using: " + url)
 
     try:
         fp = helpers.urlopen(url)
@@ -65,7 +66,7 @@ def scan(cpv, url):
         if helpers.version_filtered(cp, ver, pv):
             continue
         url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
-        ret.append((url, pv))
+        ret.append((url, pv, HANDLER_NAME))
 
     return ret