src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

euscan: Redesigning the handlers layout

Browse Source 
Signed-off-by: volpino <fox91@anche.no>
This commit is contained in:
volpino 2012-07-26 10:44:10 +02:00
parent 36e1aa6d12
commit 12bf1fc75b
13 changed files with 168 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bin/euscan
View File

@ -130,9 +130,9 @@ def print_usage(_error=None, help=None):
" - Ignore non-stable versions only if current\n" +
" " * 38 + "version is stable", file=out)
print(yellow(" --mirror") +
" - use mirror:// URIs", file=out)
" - use mirror:// URIs", file=out)
print(yellow(" --ebuild-uri") +
" - use ebuild variables in URIs", file=out)
" - use ebuild variables in URIs", file=out)
print(file=out)
if _error in ('packages',) or help:

103
pym/euscan/handlers/__init__.py
View File

@ -1,35 +1,86 @@
import pkgutil
# autoimport all modules in this directory and append them to handlers list
handlers = []
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
module = loader.find_module(module_name).load_module(module_name)
handlers.append(module)
# sort handlers by priority (e.g.: generic should be run lastly)
handlers = sorted(
handlers,
key=lambda handler: handler.PRIORITY,
reverse=True
)
import sys
from euscan import CONFIG, output
from euscan.handlers.package import handlers as pkg_handlers
from euscan.handlers.url import handlers as url_handlers
def find_best_handler(pkg, url):
for handler in handlers:
def find_best_pkg_handler(pkg):
"""
Find the best handler for the given package
"""
for handler in pkg_handlers:
if handler.can_handle(pkg):
return handler
return None
def find_best_url_handler(pkg, url):
"""
Find the best handler for the given url
"""
for handler in url_handlers:
if handler.can_handle(pkg, url):
return handler
return None
def scan(pkg, url):
handler = find_best_handler(pkg, url)
if handler:
return handler.scan(pkg, url)
return []
def scan(pkg, urls, on_progress=None):
"""
Scans upstream for the given package.
First tries if a package wide handler is available, then fallbacks
in url handling.
"""
pkg_handler = find_best_pkg_handler(pkg)
if pkg_handler:
if on_progress:
on_progress(increment=35)
if not CONFIG['quiet'] and not CONFIG['format']:
sys.stdout.write("\n")
def brute_force(pkg, url):
handler = find_best_handler(pkg, url)
if handler:
return handler.brute_force(pkg, url)
return []
versions = pkg_handler.scan(pkg)
if on_progress:
on_progress(increment=35)
return versions
if on_progress:
progress_available = 70
num_urls = sum([len(urls[fn]) for fn in urls])
if num_urls > 0:
progress_increment = progress_available / num_urls
else:
progress_increment = 0
versions = []
for filename in urls:
for url in urls[filename]:
if on_progress and progress_available > 0:
on_progress(increment=progress_increment)
progress_available -= progress_increment
if not CONFIG['quiet'] and not CONFIG['format']:
sys.stdout.write("\n")
output.einfo("SRC_URI is '%s'" % url)
if '://' not in url:
output.einfo("Invalid url '%s'" % url)
continue
try:
url_handler = find_best_url_handler(pkg, url)
versions.extend(url_handler.scan(pkg, url))
except Exception as e:
output.ewarn(
"Handler failed: [%s] %s" %
(e.__class__.__name__, e.message)
)
if versions and CONFIG['oneshot']:
break
if on_progress and progress_available > 0:
on_progress(increment=progress_available)
return versions

19
pym/euscan/handlers/package/__init__.py Normal file
View File

@ -0,0 +1,19 @@
"""
Package wide handlers for scanning upstream
"""
import pkgutil
handlers = []
# autoimport all modules in this directory and append them to handlers list
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
module = loader.find_module(module_name).load_module(module_name)
handlers.append(module)
# sort handlers by priority
handlers = sorted(
handlers,
key=lambda handler: handler.PRIORITY,
reverse=True
)

14
pym/euscan/handlers/watch.py → pym/euscan/handlers/package/watch.py
View File

@ -3,7 +3,7 @@ import urllib2
import portage
from euscan.handlers import generic
from euscan.handlers.url import generic
from euscan import output, helpers
PRIORITY = 100
@ -15,7 +15,7 @@ CONFIDENCE = 100.0
is_pattern = r"\([^\/]+\)"
def can_handle(pkg, url):
def can_handle(pkg):
try:
return pkg.metadata._xml_tree.find("upstream").find("watch") \
is not None
@ -116,7 +116,7 @@ def handle_directory_patterns(base, file_pattern):
for _, path in scan_data]
def scan(pkg, url):
def scan(pkg):
output.einfo("Using watch data")
cp, ver, rev = portage.pkgsplit(pkg.cpv)
@ -126,18 +126,14 @@ def scan(pkg, url):
if not re.search(is_pattern, base):
steps = [(base, file_pattern)]
res = generic.scan_directory_recursive(
cp, ver, rev, "", steps, url
cp, ver, rev, "", steps, base
)
else:
res = []
for step in handle_directory_patterns(base, file_pattern):
res += generic.scan_directory_recursive(
cp, ver, rev, "", [step], url
cp, ver, rev, "", [step], base
)
results += clean_results(res, versionmangle, urlmangle)
return results
def brute_force(pkg, url):
return []

19
pym/euscan/handlers/url/__init__.py Normal file
View File

@ -0,0 +1,19 @@
"""
Url wide handlers for scanning upstream
"""
import pkgutil
handlers = []
# autoimport all modules in this directory and append them to handlers list
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
module = loader.find_module(module_name).load_module(module_name)
handlers.append(module)
# sort handlers by priority
handlers = sorted(
handlers,
key=lambda handler: handler.PRIORITY,
reverse=True
)

4
pym/euscan/handlers/cpan.py → pym/euscan/handlers/url/cpan.py
View File

@ -131,7 +131,3 @@ def scan(pkg, url):
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret
def brute_force(pkg, url):
return []

55
pym/euscan/handlers/generic.py → pym/euscan/handlers/url/generic.py
View File

@ -108,43 +108,50 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
def scan(pkg, url):
for bu in SCANDIR_BLACKLIST_URLS:
if re.match(bu, url):
output.einfo("%s is blacklisted by rule %s" % (url, bu))
if CONFIG["scan-dir"]:
for bu in SCANDIR_BLACKLIST_URLS:
if re.match(bu, url):
output.einfo("%s is blacklisted by rule %s" % (url, bu))
return []
resolved_url = helpers.parse_mirror(url)
if not resolved_url:
return []
resolved_url = helpers.parse_mirror(url)
if not resolved_url:
return []
cp, ver, rev = portage.pkgsplit(pkg.cpv)
cp, ver, rev = portage.pkgsplit(pkg.cpv)
# 'Hack' for _beta/_rc versions where _ is used instead of -
if ver not in resolved_url:
newver = helpers.version_change_end_sep(ver)
if newver and newver in resolved_url:
output.einfo(
"Version: using %s instead of %s" % (newver, ver)
)
ver = newver
# 'Hack' for _beta/_rc versions where _ is used instead of -
if ver not in resolved_url:
newver = helpers.version_change_end_sep(ver)
if newver and newver in resolved_url:
template = helpers.template_from_url(resolved_url, ver)
if '${' not in template:
output.einfo(
"Version: using %s instead of %s" % (newver, ver)
"Url doesn't seems to depend on version: %s not found in %s" %
(ver, resolved_url)
)
ver = newver
return []
else:
output.einfo("Scanning: %s" % template)
template = helpers.template_from_url(resolved_url, ver)
if '${' not in template:
output.einfo(
"Url doesn't seems to depend on version: %s not found in %s" %
(ver, resolved_url)
)
return []
else:
output.einfo("Scanning: %s" % template)
steps = helpers.generate_scan_paths(template)
ret = scan_directory_recursive(cp, ver, rev, "", steps, url)
steps = helpers.generate_scan_paths(template)
ret = scan_directory_recursive(cp, ver, rev, "", steps, url)
if not ret:
brute_force(pkg, url)
return ret
def brute_force(pkg, url):
if CONFIG["brute-force"] == 0:
return []
cp, ver, rev = portage.pkgsplit(pkg.cpv)
url = helpers.parse_mirror(url)

4
pym/euscan/handlers/github.py → pym/euscan/handlers/url/github.py
View File

@ -52,7 +52,3 @@ def scan(pkg, url):
if helpers.version_filtered(cp, ver, pv):
continue
yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE)
def brute_force(pkg, url):
return []

15
pym/euscan/handlers/kde.py → pym/euscan/handlers/url/kde.py
View File

@ -1,4 +1,4 @@
from euscan.handlers import generic
from euscan.handlers.url import generic
PRIORITY = 90
@ -29,14 +29,11 @@ def scan(pkg, url):
url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/')
results += generic.scan(pkg.cpv, url)
return clean_results(results)
if not results: # if nothing was found go brute forcing
results = generic.brute_force(pkg.cpv, url)
def brute_force(pkg, url):
results = generic.brute_force(pkg.cpv, url)
if url.startswith('mirror://kde/unstable/'):
url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/')
results += generic.brute_force(pkg.cpv, url)
if url.startswith('mirror://kde/unstable/'):
url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/')
results += generic.brute_force(pkg.cpv, url)
return clean_results(results)

4
pym/euscan/handlers/php.py → pym/euscan/handlers/url/php.py
View File

@ -70,7 +70,3 @@ def scan(pkg, url):
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret
def brute_force(pkg, url):
return []

4
pym/euscan/handlers/pypi.py → pym/euscan/handlers/url/pypi.py
View File

@ -52,7 +52,3 @@ def scan(pkg, url):
ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
return ret
def brute_force(pkg, url):
return []

4
pym/euscan/handlers/rubygem.py → pym/euscan/handlers/url/rubygem.py
View File

@ -71,7 +71,3 @@ def scan(pkg, url):
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret
def brute_force(pkg, url):
return []

72
pym/euscan/scan.py
View File

@ -44,60 +44,6 @@ def filter_versions(cp, versions):
]
def scan_upstream_urls(pkg, urls, on_progress):
versions = []
if on_progress:
progress_available = 70
num_urls = sum([len(urls[fn]) for fn in urls])
if num_urls > 0:
progress_increment = progress_available / num_urls
else:
progress_increment = 0
for filename in urls:
for url in urls[filename]:
if on_progress and progress_available > 0:
on_progress(increment=progress_increment)
progress_available -= progress_increment
if not CONFIG['quiet'] and not CONFIG['format']:
pp.uprint()
output.einfo("SRC_URI is '%s'" % url)
if '://' not in url:
output.einfo("Invalid url '%s'" % url)
continue
# Try normal scan
if CONFIG["scan-dir"]:
try:
versions.extend(handlers.scan(pkg, url))
except Exception as e:
output.ewarn("Handler failed: [%s] %s"
% (e.__class__.__name__, e.message))
if versions and CONFIG['oneshot']:
break
# Brute Force
if CONFIG["brute-force"] > 0:
versions.extend(handlers.brute_force(pkg, url))
if versions and CONFIG['oneshot']:
break
cp, ver, rev = portage.pkgsplit(pkg.cpv)
result = filter_versions(cp, versions)
if on_progress and progress_available > 0:
on_progress(increment=progress_available)
return result
# gentoolkit stores PORTDB, so even if we modify it to add an overlay
# it will still use the old dbapi
def reload_gentoolkit():
@ -120,7 +66,6 @@ def scan_upstream(query, on_progress=None):
"""
Scans the upstream searching new versions for the given query
"""
matches = []
if query.endswith(".ebuild"):
@ -188,10 +133,6 @@ def scan_upstream(query, on_progress=None):
output.metadata("description", pkg.environment("DESCRIPTION"))
cpv = pkg.cpv
_, _, ver, _ = portage.catpkgsplit(cpv)
is_current_version_stable = is_version_stable(ver)
metadata = {
"EAPI": portage.settings["EAPI"],
"SRC_URI": pkg.environment("SRC_URI", False),
@ -212,15 +153,20 @@ def scan_upstream(query, on_progress=None):
else:
urls = alist
# output scan time for formatted output
scan_time = (datetime.now() - start_time).total_seconds()
output.metadata("scan_time", scan_time, show=False)
versions = handlers.scan(pkg, urls, on_progress)
result = scan_upstream_urls(pkg, urls, on_progress)
cp, ver, rev = portage.pkgsplit(pkg.cpv)
result = filter_versions(cp, versions)
if on_progress:
on_progress(increment=10)
# output scan time for formatted output
scan_time = (datetime.now() - start_time).total_seconds()
output.metadata("scan_time", scan_time, show=False)
is_current_version_stable = is_version_stable(ver)
if len(result) > 0:
if not (CONFIG['format'] or CONFIG['quiet']):
print("\n", file=sys.stderr)