From a31b6fa4a8e5361e99745cdf34f8f63fb85382bf Mon Sep 17 00:00:00 2001 From: volpino Date: Tue, 7 Aug 2012 16:42:27 +0200 Subject: [PATCH] euscan: Adding new handlers: google-code, berlios, sourceforge Signed-off-by: volpino --- pym/euscan/handlers/berlios.py | 49 ++++++++++++++++++++++++++++++ pym/euscan/handlers/generic.py | 4 +-- pym/euscan/handlers/github.py | 2 ++ pym/euscan/handlers/google_code.py | 37 ++++++++++++++++++++++ pym/euscan/handlers/sourceforge.py | 34 +++++++++++++++++++++ pym/euscan/handlers/url.py | 13 +++++--- 6 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 pym/euscan/handlers/berlios.py create mode 100644 pym/euscan/handlers/google_code.py create mode 100644 pym/euscan/handlers/sourceforge.py diff --git a/pym/euscan/handlers/berlios.py b/pym/euscan/handlers/berlios.py new file mode 100644 index 0000000..9667e77 --- /dev/null +++ b/pym/euscan/handlers/berlios.py @@ -0,0 +1,49 @@ +import re +import urllib + +import portage + +from euscan.helpers import regex_from_template +from euscan.handlers.url import process_scan as url_scan +from euscan import output + +HANDLER_NAME = "berlios" +CONFIDENCE = 90 +PRIORITY = 95 + + +def can_handle(pkg, url=None): + return url and "berlios.de/" in url + + +def scan_url(pkg, url, options): + output.einfo("Using BerliOS handler") + + cp, ver, rev = portage.pkgsplit(pkg.cpv) + + project, filename = re.search(r"berlios.de/(.+)/(.+)", url).groups() + + project_page = "http://developer.berlios.de/projects/%s" % project + content = urllib.urlopen(project_page).read() + + project_id = re.search( + r"/project/filelist.php\?group_id=(\d+)", + content + ).group(1) + + base_url = ( + "http://developer.berlios.de/project/filelist.php?group_id=%s" % + project_id + ) + + file_pattern = regex_from_template( + filename.replace(ver, "${PV}") + ) + + result = url_scan(pkg, base_url, file_pattern) + + ret = [] + for found_url, pv, _, _ in result: + found_url = found_url.replace("prdownload", "download") + ret.append((found_url, pv, HANDLER_NAME, CONFIDENCE)) + return ret diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/generic.py index c00ee24..fd82c71 100644 --- a/pym/euscan/handlers/generic.py +++ b/pym/euscan/handlers/generic.py @@ -11,8 +11,8 @@ except ImportError: import portage -from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \ - BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS, output, helpers, mangling +from euscan import output, helpers, mangling, CONFIG, SCANDIR_BLACKLIST_URLS, \ + BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS HANDLER_NAME = "generic" CONFIDENCE = 45 diff --git a/pym/euscan/handlers/github.py b/pym/euscan/handlers/github.py index dfe2cee..7c4dc69 100644 --- a/pym/euscan/handlers/github.py +++ b/pym/euscan/handlers/github.py @@ -14,12 +14,14 @@ PRIORITY = 90 def can_handle(pkg, url=None): return url and url.startswith('mirror://github/') + def guess_package(cp, url): match = re.search('^mirror://github/(.*?)/(.*?)/(.*)$', url) assert(match) return (match.group(1), match.group(2), match.group(3)) + def scan_url(pkg, url, options): 'http://developer.github.com/v3/repos/downloads/' diff --git a/pym/euscan/handlers/google_code.py b/pym/euscan/handlers/google_code.py new file mode 100644 index 0000000..a5cb037 --- /dev/null +++ b/pym/euscan/handlers/google_code.py @@ -0,0 +1,37 @@ +import re +import portage + +from euscan import output +from euscan.helpers import regex_from_template +from euscan.handlers.url import process_scan as url_scan + +HANDLER_NAME = "google-code" +CONFIDENCE = 90 +PRIORITY = 90 + + +package_name_regex = r"http://(.+).googlecode.com/files/.+" + + +def can_handle(pkg, url=None): + return url and re.match(package_name_regex, url) + + +def scan_url(pkg, url, options): + output.einfo("Using Google Code handler") + + cp, ver, rev = portage.pkgsplit(pkg.cpv) + + package_name = re.match(package_name_regex, url).group(1) + base_url = "http://code.google.com/p/%s/downloads/list" % package_name + + file_pattern = regex_from_template( + url.split("/")[-1].replace(ver, "${PV}") + ) + + result = url_scan(pkg, base_url, file_pattern) + + ret = [] + for found_url, pv, _, _ in result: + ret.append((found_url, pv, HANDLER_NAME, CONFIDENCE)) + return ret diff --git a/pym/euscan/handlers/sourceforge.py b/pym/euscan/handlers/sourceforge.py new file mode 100644 index 0000000..a9f08b5 --- /dev/null +++ b/pym/euscan/handlers/sourceforge.py @@ -0,0 +1,34 @@ +import re + +import portage + +from euscan.helpers import regex_from_template +from euscan.handlers.url import process_scan as url_scan +from euscan import output + +HANDLER_NAME = "sourceforge" +CONFIDENCE = 90 +PRIORITY = 95 + + +def can_handle(pkg, url=None): + return url and "sourceforge.net/" in url + + +def scan_url(pkg, url, options): + output.einfo("Using SourceForge handler") + + cp, ver, rev = portage.pkgsplit(pkg.cpv) + + project, filename = re.search("sourceforge.net/(.+)/(.+)", url).groups() + base_url = "http://qa.debian.org/watch/sf.php/%s" % project + file_pattern = regex_from_template( + filename.replace(ver, "${PV}") + ) + + result = url_scan(pkg, base_url, file_pattern) + + ret = [] + for found_url, pv, _, _ in result: + ret.append((found_url, pv, HANDLER_NAME, CONFIDENCE)) + return ret diff --git a/pym/euscan/handlers/url.py b/pym/euscan/handlers/url.py index 66d2f94..29e7ed4 100644 --- a/pym/euscan/handlers/url.py +++ b/pym/euscan/handlers/url.py @@ -77,13 +77,12 @@ def read_options(options): return base, file_pattern -def scan_pkg(pkg, options): - output.einfo("Using watch data") +def process_scan(pkg, base, file_pattern, options=None): + if options is None: + options = {} cp, ver, rev = pkg.cp, pkg.version, pkg.revision - base, file_pattern = read_options(options) - results = [] if not re.search(is_pattern, base): steps = [(base, file_pattern)] @@ -97,3 +96,9 @@ def scan_pkg(pkg, options): ) return results + + +def scan_pkg(pkg, options): + output.einfo("Using watch data") + base, file_pattern = read_options(options) + return process_scan(pkg, base, file_pattern, options)