diff --git a/README b/README index 730deb4..378e6a4 100644 --- a/README +++ b/README @@ -88,6 +88,15 @@ tree to avoid messing with your system. ### Installation +Install requirements from PyPI using + + $ python setup.py develop + +Extra dependencies: + + * portage python api + * rrdtool[python] + Like any django web app, just start by editing settings.py and then run these two commands. diff --git a/bin/euscan b/bin/euscan index 19c9ee5..52606d6 100755 --- a/bin/euscan +++ b/bin/euscan @@ -34,11 +34,12 @@ from euscan.scan import scan_upstream """ Globals """ + def setupSignals(): """ This block ensures that ^C interrupts are handled quietly. """ import signal - def exithandler(signum,frame): + def exithandler(signum, frame): signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) print () @@ -51,10 +52,10 @@ def setupSignals(): def printVersion(): """Output the version info.""" - print( "%s (%s) - %s" \ + print("%s (%s) - %s" \ % (__productname__, __version__, __description__)) print() - print("Author: %s <%s>" % (__author__,__email__)) + print("Author: %s <%s>" % (__author__, __email__)) print("Copyright 2011 Gentoo Foundation") print("Distributed under the terms of the GNU General Public License v2") @@ -68,45 +69,49 @@ def printUsage(_error=None, help=None): out = sys.stderr if not _error in ('global-options', 'packages',): _error = None - if not _error and not help: help = 'all' + if not _error and not help: + help = 'all' if _error in ('global-options',): - print( pp.error("Wrong option on command line."), file=out) - print( file=out) + print(pp.error("Wrong option on command line."), file=out) + print(file=out) if _error in ('packages',): - print( pp.error("You need to specify exactly one package."), file=out) - print( file=out) - print( white("Usage:"), file=out) + print(pp.error("You need to specify exactly one package."), file=out) + print(file=out) + print(white("Usage:"), file=out) if _error in ('global-options', 'packages',) or help == 'all': - print( " "+turquoise(__productname__), + print(" " + turquoise(__productname__), yellow("[options]"), green(" [ [...]]"), file=out) if _error in ('global-options',) or help == 'all': - print( " "+turquoise(__productname__), + print(" " + turquoise(__productname__), yellow("[--help, --version]"), file=out) print(file=out) if _error in ('global-options',) or help: - print( "Available ", yellow("options")+":", file=out) - print( yellow(" -C, --nocolor")+ - " - turn off colors on output", file=out) - print( yellow(" -q, --quiet")+ - " - be as quiet as possible", file=out) - print( yellow(" -h, --help")+ \ - " - display the help screen", file=out) - print( yellow(" -V, --version")+ - " - display version info", file=out) - print( file=out) - print( yellow(" -1, --oneshot")+ - " - stop as soon as a new version is found", file=out) - print( yellow(" -b, --brute-force=")+ - " - define the brute force "+yellow("")+" (default: 2)\n" + - " " * 29 + "bigger levels will generate more versions numbers\n" + - " " * 29 + "0 means disabled", file=out) - print( file=out) + print("Available ", yellow("options") + ":", file=out) + print(yellow(" -C, --nocolor") + + " - turn off colors on output", file=out) + print(yellow(" -q, --quiet") + + " - be as quiet as possible", file=out) + print(yellow(" -h, --help") + + " - display the help screen", file=out) + print(yellow(" -V, --version") + + " - display version info", file=out) + print(file=out) + print(yellow(" -1, --oneshot") + + " - stop as soon as a new version is found", + file=out) + print(yellow(" -b, --brute-force=") + + " - define the brute force " + yellow("") + + " (default: 2)\n" + + " " * 29 + "bigger levels will generate more versions numbers\n" + + " " * 29 + "0 means disabled", file=out) + print(file=out) if _error in ('packages',) or help: - print( green(" package")+ - " - the packages (or ebuilds) you want to scan", file=out) - print( file=out) + print(green(" package") + + " - the packages (or ebuilds) you want to scan", + file=out) + print(file=out) '''print( "More detailed instruction can be found in", turquoise("`man %s`" % __productname__), file=out)''' @@ -115,6 +120,7 @@ class ParseArgsException(Exception): """For parseArgs() -> main() communications.""" def __init__(self, value): self.value = value + def __str__(self): return repr(self.value) @@ -130,7 +136,7 @@ def parseArgs(): return_code = True for o, a in opts: if o in ("-h", "--help"): - raise ParseArgsException('help') + raise ParseArgsException('help') elif o in ("-V", "--version"): raise ParseArgsException('version') elif o in ("-C", "--nocolor"): @@ -151,7 +157,7 @@ def parseArgs(): return return_code ' here are the different allowed command line options (getopt args) ' - getopt_options = {'short':{}, 'long':{}} + getopt_options = {'short': {}, 'long': {}} getopt_options['short']['global'] = "hVCqv1b:" getopt_options['long']['global'] = ["help", "version", "nocolor", "quiet", "verbose", "oneshot", "brute-force="] @@ -164,19 +170,20 @@ def parseArgs(): try: opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts) except: - raise ParseArgsException(opts_mode+'-options') + raise ParseArgsException(opts_mode + '-options') ' set options accordingly ' optionSwitch(opts) if len(args) < 1: - raise ParseArgsException('packages') + raise ParseArgsException('packages') return args + def main(): """Parse command line and execute all actions.""" - CONFIG['nocolor'] = (port_settings["NOCOLOR"] in ('yes','true') + CONFIG['nocolor'] = (port_settings["NOCOLOR"] in ('yes', 'true') or not sys.stdout.isatty()) if CONFIG['nocolor']: pp.output.nocolor() @@ -212,11 +219,16 @@ def main(): for candidate in pkgs: print(candidate) - from os.path import basename # To get the short name + from os.path import basename # To get the short name print(file=sys.stderr) - print(pp.error("The short ebuild name '%s' is ambiguous. Please specify" % basename(pkgs[0])), - file=sys.stderr, end="") + print( + pp.error( + "The short ebuild name '%s' is ambiguous. Please specify" \ + % basename(pkgs[0]) + ), + file=sys.stderr, end="" + ) pp.die(1, "one of the above fully-qualified ebuild names instead.") except GentoolkitException as err: pp.die(1, '%s: %s' % (package, str(err))) @@ -233,12 +245,12 @@ def main(): + pp.path(" %s" % url)) else: print (pp.cpv("%s-%s" % (cp, version)) - + ": " + pp.path(url)) + + ": " + pp.path(url)) if not len(ret) and not CONFIG['quiet']: print (pp.warn("Didn't find any new version, " + "check package's homepage for " - + "more informations")); + + "more informations")) if __name__ == "__main__": @@ -246,6 +258,6 @@ if __name__ == "__main__": setupSignals() main() except KeyboardInterrupt: - print( "Aborted.") + print("Aborted.") sys.exit(errno.EINTR) sys.exit(0) diff --git a/euscanwww/djeuscan/admin.py b/euscanwww/djeuscan/admin.py index 3b7d8e9..563cd2c 100644 --- a/euscanwww/djeuscan/admin.py +++ b/euscanwww/djeuscan/admin.py @@ -1,11 +1,14 @@ -from models import * +from models import Package, Version, VersionLog, EuscanResult, Log, WorldLog, \ + CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer from django.contrib import admin admin.site.register(Herd) admin.site.register(Maintainer) + class PackageAdmin(admin.ModelAdmin): search_fields = ('category', 'name') + admin.site.register(Package, PackageAdmin) admin.site.register(Version) diff --git a/euscanwww/djeuscan/api/emitters.py b/euscanwww/djeuscan/api/emitters.py index f7b6ab7..1f7a63a 100644 --- a/euscanwww/djeuscan/api/emitters.py +++ b/euscanwww/djeuscan/api/emitters.py @@ -1,15 +1,16 @@ -from piston.emitters import Emitter, XMLEmitter +from piston.emitters import XMLEmitter + class EuscanXMLEmitter(XMLEmitter): _parent = [] _known_parents = { - 'vlog' : 'version', - 'herds' : 'herd', - 'maintainers' : 'maintainer', - 'packaged' : 'version', - 'upstream' : 'version', - 'packages' : 'package', - 'categories' : 'category' + 'vlog': 'version', + 'herds': 'herd', + 'maintainers': 'maintainer', + 'packaged': 'version', + 'upstream': 'version', + 'packages': 'package', + 'categories': 'category' } def _push_parent(self, parent): diff --git a/euscanwww/djeuscan/api/handlers.py b/euscanwww/djeuscan/api/handlers.py index 1297edb..2cfa53a 100644 --- a/euscanwww/djeuscan/api/handlers.py +++ b/euscanwww/djeuscan/api/handlers.py @@ -1,24 +1,26 @@ -from piston.handler import AnonymousBaseHandler, BaseHandler -from piston.utils import rc, HttpStatusCode +from piston.handler import AnonymousBaseHandler +from piston.utils import rc from django.db.models import Sum, Max from django.core.exceptions import ObjectDoesNotExist from django.forms.models import model_to_dict -from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, VersionLog -from djeuscan.forms import WorldForm, PackagesForm +from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \ + VersionLog # replace default XMLEmitter with ours from piston.emitters import Emitter from emitters import EuscanXMLEmitter Emitter.register('xml', EuscanXMLEmitter, 'text/xml; charset=utf-8') + def xint(i): try: return int(i) except: return 0 + def renameFields(vqs, fields): ret = [] for n in vqs: @@ -29,6 +31,7 @@ def renameFields(vqs, fields): ret.append(n) return ret + class catch_and_return(object): def __init__(self, err, response): self.err = err @@ -42,6 +45,7 @@ class catch_and_return(object): return self.response return wrapper + # /api/1.0/ class RootHandler(AnonymousBaseHandler): allowed_methods = ('GET',) @@ -49,52 +53,70 @@ class RootHandler(AnonymousBaseHandler): def read(self, request): return {'api-version': '1.0'} + # /api/1.0/statistics class StatisticsHandler(AnonymousBaseHandler): allowed_methods = ('GET',) def read(self, request): + _aggr = Package.objects.aggregate data = {} - data['n_packaged'] = xint(Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum']) - data['n_overlay'] = xint(Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum']) - data['n_versions'] = xint(Package.objects.aggregate(Sum('n_versions'))['n_versions__sum']) - data['n_upstream'] = data['n_versions'] - data['n_packaged'] - data['n_overlay'] + data['n_packaged'] = xint(_aggr(Sum('n_packaged'))['n_packaged__sum']) + data['n_overlay'] = xint(_aggr(Sum('n_overlay'))['n_overlay__sum']) + data['n_versions'] = xint(_aggr(Sum('n_versions'))['n_versions__sum']) + data['n_upstream'] = data['n_versions'] - data['n_packaged'] - \ + data['n_overlay'] data['n_packages'] = Package.objects.count() data['n_herds'] = Herd.objects.count() data['n_maintainers'] = Maintainer.objects.count() - data['last_scan'] = EuscanResult.objects.get(id=EuscanResult.objects.aggregate(Max('id'))['id__max']).datetime + data['last_scan'] = EuscanResult.objects.get( + id=EuscanResult.objects.aggregate(Max('id'))['id__max'] + ).datetime return data + # /api/1.0/maintainers class MaintainersHandler(AnonymousBaseHandler): allowed_methods = ('GET',) def read(self, request): maintainers = Package.objects.filter(maintainers__isnull=False) - maintainers = maintainers.values('maintainers__id', 'maintainers__name', 'maintainers__email') + maintainers = maintainers.values( + 'maintainers__id', + 'maintainers__name', + 'maintainers__email' + ) maintainers = maintainers.annotate(n_packaged=Sum('n_packaged'), n_overlay=Sum('n_overlay'), n_versions=Sum('n_versions')) - maintainers = renameFields(maintainers, [('maintainers__id', 'id'), - ('maintainers__name', 'name'), - ('maintainers__email', 'email')]) - return { 'maintainers' : maintainers } + maintainers = renameFields( + maintainers, + [('maintainers__id', 'id'), + ('maintainers__name', 'name'), + ('maintainers__email', 'email')] + ) + return {'maintainers': maintainers} + # /api/1.0/herds class HerdsHandler(AnonymousBaseHandler): allowed_methods = ('GET',) def read(self, request): - # FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of 'INNER JOIN' + # FIXME: optimize the query, it uses 'LEFT OUTER JOIN' + # instead of 'INNER JOIN' herds = Package.objects.filter(herds__isnull=False) - herds = herds.values('herds__herd').annotate(n_packaged=Sum('n_packaged'), - n_overlay=Sum('n_overlay'), - n_versions=Sum('n_versions')) + herds = herds.values('herds__herd').annotate( + n_packaged=Sum('n_packaged'), + n_overlay=Sum('n_overlay'), + n_versions=Sum('n_versions') + ) herds = renameFields(herds, [('herds__herd', 'herd')]) - return { 'herds' : herds } + return {'herds': herds} + # /api/1.0/categories class CategoriesHandler(AnonymousBaseHandler): @@ -106,7 +128,8 @@ class CategoriesHandler(AnonymousBaseHandler): n_overlay=Sum('n_overlay'), n_versions=Sum('n_versions')) - return { 'categories' : categories } + return {'categories': categories} + # /api/1.0/packages/by-maintainer/ # /api/1.0/packages/by-category/ @@ -125,17 +148,21 @@ class PackagesHandler(AnonymousBaseHandler): if 'category' in kwargs: packages = Package.objects.filter(category=kwargs['category']) - data = { 'category' : kwargs['category'] } + data = {'category': kwargs['category']} elif 'herd' in kwargs: herd = Herd.objects.get(herd=kwargs['herd']) packages = Package.objects.filter(herds__id=herd.id) - data = { 'herd' : herd } + data = {'herd': herd} elif 'maintainer_id' in kwargs: maintainer = Maintainer.objects.get(id=kwargs['maintainer_id']) packages = Package.objects.filter(maintainers__id=maintainer.id) - data = { 'maintainer' : maintainer } + data = {'maintainer': maintainer} - packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream') + packages = packages.select_related( + 'last_version_gentoo', + 'last_version_overlay', + 'last_version_upstream' + ) data['packages'] = packages if not data: @@ -143,6 +170,7 @@ class PackagesHandler(AnonymousBaseHandler): return data + # /api/1.0/package/ class PackageHandler(AnonymousBaseHandler): allowed_methods = ('GET',) @@ -152,7 +180,8 @@ class PackageHandler(AnonymousBaseHandler): package = Package.objects.get(category=category, name=package) package.homepages = package.homepage.split(' ') versions = Version.objects.filter(package=package) - log = EuscanResult.objects.filter(package=package).order_by('-datetime')[:1] + log = EuscanResult.objects.filter(package=package).\ + order_by('-datetime')[:1] log = log[0] if log else None vlog = VersionLog.objects.filter(package=package).order_by('-id') @@ -166,7 +195,8 @@ class PackageHandler(AnonymousBaseHandler): version_log = [] for v in vlog: - v = model_to_dict(v, ['version', 'revision', 'slot', 'overlay', 'datetime', 'action']) + v = model_to_dict(v, ['version', 'revision', 'slot', 'overlay', + 'datetime', 'action']) if v['action'] == VersionLog.VERSION_ADDED: v['action'] = 'added' if v['action'] == VersionLog.VERSION_REMOVED: @@ -177,7 +207,10 @@ class PackageHandler(AnonymousBaseHandler): packaged = [] for version in versions: unpackaged = not version.packaged - version = model_to_dict(version, ['version', 'revision', 'slot', 'overlay', 'urls']) + version = model_to_dict( + version, + ['version', 'revision', 'slot', 'overlay', 'urls'] + ) if unpackaged: upstream.append(version) else: diff --git a/euscanwww/djeuscan/api/urls.py b/euscanwww/djeuscan/api/urls.py index 8fe4b4b..ffab482 100644 --- a/euscanwww/djeuscan/api/urls.py +++ b/euscanwww/djeuscan/api/urls.py @@ -1,7 +1,8 @@ -from django.conf.urls.defaults import * +from django.conf.urls.defaults import url, patterns from piston.resource import Resource -from handlers import * +from handlers import RootHandler, StatisticsHandler, HerdsHandler, \ + CategoriesHandler, MaintainersHandler, PackagesHandler, PackageHandler root_handler = Resource(handler=RootHandler) statistics_handler = Resource(handler=StatisticsHandler) @@ -12,16 +13,25 @@ packages_handler = Resource(handler=PackagesHandler) package_handler = Resource(handler=PackageHandler) urlpatterns = patterns('djeuscan.api.views', - url(r'^1.0/statistics\.(?P.+)$', statistics_handler, name='api.views.statistics'), - url(r'^1.0/herds\.(?P.+)$', herds_handler, name='api.views.herds'), - url(r'^1.0/categories\.(?P.+)$', categories_handler, name='api.views.categories'), - url(r'^1.0/maintainers\.(?P.+)$', maintainers_handler, name='api.views.maintainers'), + url(r'^1.0/statistics\.(?P.+)$', statistics_handler, + name='api.views.statistics'), + url(r'^1.0/herds\.(?P.+)$', herds_handler, + name='api.views.herds'), + url(r'^1.0/categories\.(?P.+)$', categories_handler, + name='api.views.categories'), + url(r'^1.0/maintainers\.(?P.+)$', maintainers_handler, + name='api.views.maintainers'), - url(r'^1.0/packages/by-maintainer/(?P\d+)\.(?P.+)$', packages_handler, name='api.views.packages'), - url(r'^1.0/packages/by-herd/(?P[\@\{\}\w+.-]*)\.(?P.+)?$', packages_handler, name='api.views.packages'), - url(r'^1.0/packages/by-category/(?P[\w+][\w+.-]*)\.(?P.+)?$', packages_handler, name='api.views.packages'), + url(r'^1.0/packages/by-maintainer/(?P\d+)\.(?P.+)$', + packages_handler, name='api.views.packages'), + url(r'^1.0/packages/by-herd/(?P[\@\{\}\w+.-]*)\.(?P.+)?$', + packages_handler, name='api.views.packages'), + url(r'^1.0/packages/by-category/(?P[\w+][\w+.-]*)\.(?P.+)?$', + packages_handler, name='api.views.packages'), - url(r'^1.0/package/(?P[\w+][\w+.-]*)/(?P[\w+][\w+.-]*)\.(?P.+)$', package_handler, name='api.views.package'), + url(r'^1.0/package/(?P[\w+][\w+.-]*)/(?P[\w+][\w+.-]*)\.(?P.+)$', + package_handler, name='api.views.package'), - url(r'^1.0/api\.(?P.+)$', root_handler, name='api.views.root'), + url(r'^1.0/api\.(?P.+)$', + root_handler, name='api.views.root'), ) diff --git a/euscanwww/djeuscan/charts.py b/euscanwww/djeuscan/charts.py index 544c693..276e914 100644 --- a/euscanwww/djeuscan/charts.py +++ b/euscanwww/djeuscan/charts.py @@ -3,14 +3,12 @@ import time from euscanwww import settings -from django.db.models import F, Sum, Max -from djeuscan.models import Version, Package, Herd, Maintainer -from djeuscan.models import CategoryLog +from django.db.models import F, Sum +from djeuscan.models import Package import rrdtool import pylab -import matplotlib CHARTS_ROOT = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts') CHARTS_URL = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts') @@ -21,12 +19,14 @@ pylab.rcParams['axes.titlesize'] = 10.0 pylab.rcParams['xtick.labelsize'] = 8.0 pylab.rcParams['legend.fontsize'] = 8.0 + def xint(i): try: return int(i) except: return 0 + def chart_alive(name): path = os.path.join(CHARTS_ROOT, name) if not os.path.exists(path): @@ -37,6 +37,7 @@ def chart_alive(name): return False return True + def rrd_name(**kwargs): name = "" @@ -51,6 +52,7 @@ def rrd_name(**kwargs): return name + def chart_name(name, **kwargs): name = name.replace('_', '-') @@ -67,6 +69,7 @@ def chart_name(name, **kwargs): return name + ".png" + def getpackages(**kwargs): packages = Package.objects @@ -79,6 +82,7 @@ def getpackages(**kwargs): return packages + def cached_pylab_chart(f): def new_f(*args, **kwds): name = chart_name(f.func_name, **kwds) @@ -93,14 +97,16 @@ def cached_pylab_chart(f): new_f.func_name = f.func_name return new_f + @cached_pylab_chart def pie_versions(**kwargs): - n_packaged = xint(getpackages(**kwargs).aggregate(Sum('n_packaged'))['n_packaged__sum']) - n_overlay = xint(getpackages(**kwargs).aggregate(Sum('n_overlay'))['n_overlay__sum']) - n_versions = xint(getpackages(**kwargs).aggregate(Sum('n_versions'))['n_versions__sum']) + gpk = getpackages(**kwargs) + n_packaged = xint(gpk.aggregate(Sum('n_packaged'))['n_packaged__sum']) + n_overlay = xint(gpk.aggregate(Sum('n_overlay'))['n_overlay__sum']) + n_versions = xint(gpk.aggregate(Sum('n_versions'))['n_versions__sum']) n_upstream = n_versions - n_packaged - n_overlay - pylab.figure(1, figsize=(3.5,3.5)) + pylab.figure(1, figsize=(3.5, 3.5)) if n_overlay: labels = 'Gentoo', 'Overlays', 'Upstream' @@ -111,35 +117,43 @@ def pie_versions(**kwargs): fracs = [n_packaged, n_upstream] colors = '#008000', '#FF0000' - pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', shadow=True) - pylab.title('Versions', bbox={'facecolor':'0.8', 'pad':5}) + pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', + shadow=True) + pylab.title('Versions', bbox={'facecolor': '0.8', 'pad': 5}) + @cached_pylab_chart def pie_packages(**kwargs): - n_packages = getpackages(**kwargs).count() - n_packages_uptodate_main = getpackages(**kwargs).filter(n_versions=F('n_packaged')).count() - n_packages_uptodate_all = getpackages(**kwargs).filter(n_versions=F('n_packaged') + F('n_overlay')).count() + gpk = getpackages(**kwargs) + n_packages = gpk.count() + n_packages_uptodate_main = gpk.filter(n_versions=F('n_packaged')).count() + n_packages_uptodate_all = gpk.filter(n_versions=F('n_packaged') + \ + F('n_overlay')).count() n_packages_outdated = n_packages - n_packages_uptodate_all - n_packages_uptodate_ovl = n_packages_uptodate_all - n_packages_uptodate_main + n_packages_uptodate_ovl = n_packages_uptodate_all - \ + n_packages_uptodate_main - pylab.figure(1, figsize=(3.5,3.5)) + pylab.figure(1, figsize=(3.5, 3.5)) if n_packages_uptodate_ovl: labels = 'Ok (gentoo)', 'Ok (overlays)', 'Outdated' - fracs = [n_packages_uptodate_main, n_packages_uptodate_ovl, n_packages_outdated] + fracs = [n_packages_uptodate_main, n_packages_uptodate_ovl, + n_packages_outdated] colors = '#008000', '#0B17FD', '#FF0000' else: labels = 'Ok (gentoo)', 'Outdated' fracs = [n_packages_uptodate_main, n_packages_outdated] colors = '#008000', '#FF0000' - pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', shadow=True) - pylab.title('Packages', bbox={'facecolor':'0.8', 'pad':5}) + pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', + shadow=True) + pylab.title('Packages', bbox={'facecolor': '0.8', 'pad': 5}) def rrd_path(name): return str(os.path.join(settings.RRD_ROOT, name + '.rrd')) + def rrd_create(name, start): path = rrd_path(name) if os.path.exists(path): @@ -156,13 +170,17 @@ def rrd_create(name, start): 'RRA:AVERAGE:0.5:5:200', 'RRA:AVERAGE:0.5:10:200') + def rrd_update(name, datetime, values): now = time.mktime(datetime.timetuple()) rrd_create(name, now) - rrdtool.update(rrd_path(name), - '%d:%d:%d:%d:%d:%d:%d' % \ - (now, values.n_packages_gentoo, values.n_packages_overlay, values.n_packages_outdated, \ - values.n_versions_gentoo, values.n_versions_overlay, values.n_versions_upstream)) + rrdtool.update( + rrd_path(name), + '%d:%d:%d:%d:%d:%d:%d' % \ + (now, values.n_packages_gentoo, values.n_packages_overlay, + values.n_packages_outdated, values.n_versions_gentoo, + values.n_versions_overlay, values.n_versions_upstream) + ) """ @@ -175,6 +193,7 @@ def rrd_update(name, datetime, values): [-J|--alt-autoscale-min] """ + def cached_rrd_chart(f): def new_f(*args, **kwds): if 'period' not in kwds: @@ -215,47 +234,55 @@ def cached_rrd_chart(f): new_f.func_name = f.func_name return new_f + @cached_rrd_chart def packages(**kwargs): - rrdtool.graph(str(kwargs['path']), - '--imgformat', 'PNG', - '--width', kwargs['width'], - '--height', kwargs['height'], - kwargs['graph-mode'], - '--color', 'CANVAS#FFFFFF00', - '--color', 'BACK#FFFFFF00', + rrdtool.graph( + str(kwargs['path']), + '--imgformat', 'PNG', + '--width', kwargs['width'], + '--height', kwargs['height'], + kwargs['graph-mode'], + '--color', 'CANVAS#FFFFFF00', + '--color', 'BACK#FFFFFF00', + + '--start', kwargs['start'], + '--end', kwargs['end'], + '--vertical-label', kwargs['vertical-label'], + '--title', kwargs['title'], + '--lower-limit', '0', + 'DEF:n_packages_gentoo=%s:n_packages_gentoo:AVERAGE' % (kwargs['rrd']), + 'DEF:n_packages_overlay=%s:n_packages_overlay:AVERAGE' % \ + (kwargs['rrd']), + 'DEF:n_packages_outdated=%s:n_packages_outdated:AVERAGE' % \ + (kwargs['rrd']), + 'LINE1.25:n_packages_gentoo#008000:Gentoo', + 'LINE1.25:n_packages_overlay#0B17FD:Overlay', + 'LINE1.25:n_packages_outdated#FF0000:Outdated' + ) - '--start', kwargs['start'], - '--end', kwargs['end'], - '--vertical-label', kwargs['vertical-label'], - '--title', kwargs['title'], - '--lower-limit', '0', - 'DEF:n_packages_gentoo=%s:n_packages_gentoo:AVERAGE' % (kwargs['rrd']), - 'DEF:n_packages_overlay=%s:n_packages_overlay:AVERAGE' % (kwargs['rrd']), - 'DEF:n_packages_outdated=%s:n_packages_outdated:AVERAGE' % (kwargs['rrd']), - 'LINE1.25:n_packages_gentoo#008000:Gentoo', - 'LINE1.25:n_packages_overlay#0B17FD:Overlay', - 'LINE1.25:n_packages_outdated#FF0000:Outdated') @cached_rrd_chart def versions(**kwargs): - rrdtool.graph(str(kwargs['path']), - '--imgformat', 'PNG', - '--width', kwargs['width'], - '--height', kwargs['height'], - kwargs['graph-mode'], - '--color', 'CANVAS#FFFFFF00', - '--color', 'BACK#FFFFFF00', - '--start', kwargs['start'], - '--end', kwargs['end'], - '--vertical-label', kwargs['vertical-label'], - '--title', kwargs['title'], - '--lower-limit', '0', - 'DEF:n_versions_gentoo=%s:n_versions_gentoo:AVERAGE' % (kwargs['rrd']), - 'DEF:n_versions_overlay=%s:n_versions_overlay:AVERAGE' % (kwargs['rrd']), - 'DEF:n_versions_outdated=%s:n_versions_upstream:AVERAGE' % (kwargs['rrd']), - 'LINE1.25:n_versions_gentoo#008000:Gentoo', - 'LINE1.25:n_versions_overlay#0B17FD:Overlay', - 'LINE1.25:n_versions_outdated#FF0000:Outdated') - - + rrdtool.graph( + str(kwargs['path']), + '--imgformat', 'PNG', + '--width', kwargs['width'], + '--height', kwargs['height'], + kwargs['graph-mode'], + '--color', 'CANVAS#FFFFFF00', + '--color', 'BACK#FFFFFF00', + '--start', kwargs['start'], + '--end', kwargs['end'], + '--vertical-label', kwargs['vertical-label'], + '--title', kwargs['title'], + '--lower-limit', '0', + 'DEF:n_versions_gentoo=%s:n_versions_gentoo:AVERAGE' % (kwargs['rrd']), + 'DEF:n_versions_overlay=%s:n_versions_overlay:AVERAGE' % \ + (kwargs['rrd']), + 'DEF:n_versions_outdated=%s:n_versions_upstream:AVERAGE' % \ + (kwargs['rrd']), + 'LINE1.25:n_versions_gentoo#008000:Gentoo', + 'LINE1.25:n_versions_overlay#0B17FD:Overlay', + 'LINE1.25:n_versions_outdated#FF0000:Outdated' + ) diff --git a/euscanwww/djeuscan/feeds.py b/euscanwww/djeuscan/feeds.py index 5099fa7..a9ea2d3 100644 --- a/euscanwww/djeuscan/feeds.py +++ b/euscanwww/djeuscan/feeds.py @@ -1,11 +1,10 @@ from django.contrib.syndication.views import Feed, FeedDoesNotExist from django.shortcuts import get_object_or_404 -from django.http import Http404 from django.utils.feedgenerator import Atom1Feed from django.core.urlresolvers import reverse -from djeuscan.models import Version, Package, Herd, Maintainer, VersionLog -from djeuscan.views import * +from djeuscan.models import Package, Herd, Maintainer, VersionLog + class BaseFeed(Feed): feed_type = Atom1Feed @@ -18,8 +17,8 @@ class BaseFeed(Feed): def item_description(self, vlog): if vlog.overlay: - txt = 'Version %s-%s [%s] of package %s ' % (vlog.version, vlog.revision, - vlog.slot, vlog.package) + txt = 'Version %s-%s [%s] of package %s ' % \ + (vlog.version, vlog.revision, vlog.slot, vlog.package) else: txt = 'Version %s of package %s ' % (vlog.version, vlog.package) if vlog.action == vlog.VERSION_REMOVED: @@ -36,8 +35,10 @@ class BaseFeed(Feed): return txt def item_link(self, vlog): - kwargs = {'category' : vlog.package.category, 'package' : vlog.package.name } - return reverse('djeuscan.views.package', kwargs=kwargs) + '#' + vlog.tag() + kwargs = {'category': vlog.package.category, + 'package': vlog.package.name} + return "%s#%s" % (reverse('djeuscan.views.package', kwargs=kwargs), + vlog.tag()) def item_pubdate(self, vlog): return vlog.datetime @@ -45,18 +46,20 @@ class BaseFeed(Feed): def item_categories(self, vlog): return [vlog.package.category] + class GlobalFeed(BaseFeed): title = "euscan" link = "/" description = "Last euscan changes" def categories(self): - categories = Package.objects.values('category').distinct(); - return [ category['category'] for category in categories ] + categories = Package.objects.values('category').distinct() + return [category['category'] for category in categories] def items(self): return VersionLog.objects.order_by('-id')[:250] + class PackageFeed(BaseFeed): feed_type = Atom1Feed @@ -67,7 +70,8 @@ class PackageFeed(BaseFeed): return "%s" % package def link(self, package): - return reverse('djeuscan.views.package', args=[package.category, package.name]) + return reverse('djeuscan.views.package', args=[package.category, + package.name]) def description(self, package): return package.description @@ -78,6 +82,7 @@ class PackageFeed(BaseFeed): def item_description(self, vlog): return '' + class MaintainerFeed(BaseFeed): feed_type = Atom1Feed @@ -91,12 +96,14 @@ class MaintainerFeed(BaseFeed): return "Last changes for %s" % maintainer def link(self, maintainer): - return reverse('djeuscan.views.maintainer', kwargs={'maintainer_id' : maintainer.id}) + return reverse('djeuscan.views.maintainer', + kwargs={'maintainer_id': maintainer.id}) def items(self, maintainer): q = VersionLog.objects.filter(package__maintainers__id=maintainer.id) return q.order_by('-id')[:50] + class HerdFeed(BaseFeed): feed_type = Atom1Feed @@ -110,12 +117,13 @@ class HerdFeed(BaseFeed): return "Last changes for %s" % herd def link(self, herd): - return reverse('djeuscan.views.herd', kwargs={'herd' : herd.herd}) + return reverse('djeuscan.views.herd', kwargs={'herd': herd.herd}) def items(self, herd): q = VersionLog.objects.filter(package__herds__id=herd.id) return q.order_by('-id')[:100] + class CategoryFeed(BaseFeed): feed_type = Atom1Feed diff --git a/euscanwww/djeuscan/forms.py b/euscanwww/djeuscan/forms.py index 3f9cad4..12dbb3f 100644 --- a/euscanwww/djeuscan/forms.py +++ b/euscanwww/djeuscan/forms.py @@ -1,7 +1,9 @@ from django import forms + class WorldForm(forms.Form): world = forms.FileField() + class PackagesForm(forms.Form): packages = forms.CharField(widget=forms.Textarea) diff --git a/euscanwww/djeuscan/management/commands/list-packages.py b/euscanwww/djeuscan/management/commands/list-packages.py index bfd3293..c1f0c38 100644 --- a/euscanwww/djeuscan/management/commands/list-packages.py +++ b/euscanwww/djeuscan/management/commands/list-packages.py @@ -1,8 +1,9 @@ from optparse import make_option -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from djeuscan.models import Package + class Command(BaseCommand): _overlays = {} help = 'List packages' diff --git a/euscanwww/djeuscan/management/commands/regen-rrds.py b/euscanwww/djeuscan/management/commands/regen-rrds.py index c8eba34..cbcbe0e 100644 --- a/euscanwww/djeuscan/management/commands/regen-rrds.py +++ b/euscanwww/djeuscan/management/commands/regen-rrds.py @@ -1,11 +1,8 @@ -import datetime - -from optparse import make_option - -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog from djeuscan import charts + class Command(BaseCommand): _overlays = {} help = 'Regenerate rrd database' @@ -15,10 +12,12 @@ class Command(BaseCommand): charts.rrd_update('world', wlog.datetime, wlog) for clog in CategoryLog.objects.all(): - charts.rrd_update('category-%s' % clog.category, clog.datetime, clog) + charts.rrd_update('category-%s' % clog.category, + clog.datetime, clog) for hlog in HerdLog.objects.all(): charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog) for mlog in MaintainerLog.objects.all(): - charts.rrd_update('maintainer-%d' % mlog.maintainer.id, mlog.datetime, mlog) + charts.rrd_update('maintainer-%d' % mlog.maintainer.id, + mlog.datetime, mlog) diff --git a/euscanwww/djeuscan/management/commands/scan-metadata.py b/euscanwww/djeuscan/management/commands/scan-metadata.py index 983f2d7..bcfbe49 100644 --- a/euscanwww/djeuscan/management/commands/scan-metadata.py +++ b/euscanwww/djeuscan/management/commands/scan-metadata.py @@ -1,19 +1,15 @@ -import subprocess -import portage import sys -import os -import re -from portage import versions from optparse import make_option from django.db.transaction import commit_on_success -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from djeuscan.models import Package, Herd, Maintainer from gentoolkit.query import Query from gentoolkit.errors import GentoolkitFatalError + class Command(BaseCommand): _overlays = {} @@ -53,16 +49,20 @@ class Command(BaseCommand): ) if not matches: - sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % query)) + sys.stderr.write( + self.style.ERROR("Unknown package '%s'\n" % query) + ) return - matches = sorted(matches) + matches = sorted(matches) pkg = matches.pop() - if '9999' in pkg.version and len(matches): + if '9999' in pkg.version and len(matches): pkg = matches.pop() if not obj: - obj, created = Package.objects.get_or_create(category=pkg.category, name=pkg.name) + obj, created = Package.objects.get_or_create( + category=pkg.category, name=pkg.name + ) else: created = False @@ -70,22 +70,34 @@ class Command(BaseCommand): obj.homepage = pkg.environment("HOMEPAGE") obj.description = pkg.environment("DESCRIPTION") except GentoolkitFatalError, err: - sys.stderr.write(self.style.ERROR("Gentoolkit fatal error: '%s'\n" % str(err))) + sys.stderr.write( + self.style.ERROR( + "Gentoolkit fatal error: '%s'\n" % str(err) + ) + ) if created and not self.options['quiet']: sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name)) if pkg.metadata: - herds = dict([(herd[0], herd) for herd in pkg.metadata.herds(True)]) - maintainers = dict([(m.email, m) for m in pkg.metadata.maintainers()]) + herds = dict( + [(herd[0], herd) for herd in pkg.metadata.herds(True)] + ) + maintainers = dict( + [(m.email, m) for m in pkg.metadata.maintainers()] + ) existing_herds = [h.herd for h in obj.herds.all()] new_herds = set(herds.keys()).difference(existing_herds) old_herds = set(existing_herds).difference(herds.keys()) existing_maintainers = [m.email for m in obj.maintainers.all()] - new_maintainers = set(maintainers.keys()).difference(existing_maintainers) - old_maintainers = set(existing_maintainers).difference(maintainers.keys()) + new_maintainers = set( + maintainers.keys()).difference(existing_maintainers + ) + old_maintainers = set( + existing_maintainers).difference(maintainers.keys() + ) for herd in obj.herds.all(): if herd.herd in old_herds: @@ -101,7 +113,9 @@ class Command(BaseCommand): for maintainer in new_maintainers: maintainer = maintainers[maintainer] - maintainer = self.store_maintainer(maintainer.name, maintainer.email) + maintainer = self.store_maintainer( + maintainer.name, maintainer.email + ) obj.maintainers.add(maintainer) obj.save() @@ -131,9 +145,12 @@ class Command(BaseCommand): if created: if not self.options['quiet']: - sys.stdout.write('+ [m] %s <%s>\n' % (name.encode('utf-8'), email)) + sys.stdout.write( + '+ [m] %s <%s>\n' % (name.encode('utf-8'), email) + ) - if not maintainer.name or name not in [maintainer.name, email, '{nil}']: + if not maintainer.name or \ + name not in [maintainer.name, email, '{nil}']: maintainer.name = name maintainer.save() diff --git a/euscanwww/djeuscan/management/commands/scan-portage.py b/euscanwww/djeuscan/management/commands/scan-portage.py index 197ed16..1d8b454 100644 --- a/euscanwww/djeuscan/management/commands/scan-portage.py +++ b/euscanwww/djeuscan/management/commands/scan-portage.py @@ -4,13 +4,13 @@ import sys import os import re -from portage import versions from optparse import make_option from django.db.transaction import commit_on_success -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from djeuscan.models import Package, Version, VersionLog + class Command(BaseCommand): _overlays = {} @@ -39,7 +39,8 @@ class Command(BaseCommand): action='store_true', dest='prefetch', default=False, - help='Prefetch all versions and packages from DB to speedup full scan process.'), + help=('Prefetch all versions and packages from DB to ' + 'speedup full scan process.')), make_option('--quiet', action='store_true', dest='quiet', @@ -49,7 +50,7 @@ class Command(BaseCommand): args = '[package package ...]' help = 'Scans portage tree and fills database' - _cache = {'packages' : {}, 'versions' : {}} + _cache = {'packages': {}, 'versions': {}} def cache_hash_package(self, category, name): return '%s/%s' % (category, name) @@ -59,22 +60,28 @@ class Command(BaseCommand): self._cache['packages'][key] = package def cache_get_package(self, category, name): - return self._cache['packages'].get(self.cache_hash_package(category, name)) + return self._cache['packages'].get( + self.cache_hash_package(category, name) + ) - def cache_hash_version(self, category, name, version, revision, slot, overlay): + def cache_hash_version(self, category, name, version, revision, slot, + overlay): key = '%s/%s-%s-r%s %s %s' % (category, name, version, revision, slot, overlay) return key - def cache_get_version(self, category, name, version, revision, slot, overlay): - key = self.cache_hash_version(category, name, version, revision, slot, overlay) + def cache_get_version(self, category, name, version, revision, slot, + overlay): + key = self.cache_hash_version(category, name, version, revision, slot, + overlay) return self._cache['versions'].get(key) def cache_store_version(self, version): - key = self.cache_hash_version(version.package.category, version.package.name, - version.version, version.revision, version.slot, - version.overlay) + key = self.cache_hash_version( + version.package.category, version.package.name, version.version, + version.revision, version.slot, version.overlay + ) self._cache['versions'][key] = version def handle(self, *args, **options): @@ -119,7 +126,8 @@ class Command(BaseCommand): cmd = ['eix', '-!'] - output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0] + output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\ + communicate()[0] output = output.strip().strip('\n').split('\n') overlay_re = re.compile(r'^\[(?P\d+)] "(?P.*?)"') @@ -139,9 +147,10 @@ class Command(BaseCommand): env = os.environ env['MY'] = "/-: []\n" - cmd = ['eix', '--format', '', '--pure-packages', '-x'] - if query: - cmd.extend(['--exact', query]) + cmd = ['eix', '--format', '', '--pure-packages', + '-x'] + if query: + cmd.extend(['--exact', query]) if self.options['all']: if not self.options['quiet']: @@ -151,7 +160,8 @@ class Command(BaseCommand): if not self.options['quiet']: self.stdout.write('done\n') - output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0] + output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\ + communicate()[0] output = output.strip().strip('\n') if len(output) == 0: @@ -166,17 +176,23 @@ class Command(BaseCommand): else: Package.objects.filter(name=query).delete() else: - sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % query)) + sys.stderr.write( + self.style.ERROR( + "Unknown package '%s'\n" % query + ) + ) return - output = output.split('\n') + output = output.split('\n') packages = {} - line_re = re.compile(r'^(?P.*?):(?P.*?) \[(?P.*?)\]$') + line_re = re.compile( + r'^(?P.*?):(?P.*?) \[(?P.*?)\]$' + ) package = None - for line in output: + for line in output: match = line_re.match(line) if not match: @@ -190,7 +206,8 @@ class Command(BaseCommand): packages['%s/%s' % (cat, pkg)] = True - if not package or not (cat == package.category and pkg == package.name): + if not package or not \ + (cat == package.category and pkg == package.name): package = self.store_package(cat, pkg) self.store_version(package, cpv, slot, overlay) @@ -208,16 +225,23 @@ class Command(BaseCommand): obj = self.cache_get_package(cat, pkg) if not obj: - obj, created = Package.objects.get_or_create(category=cat, name=pkg) + obj, created = Package.objects.get_or_create( + category=cat, + name=pkg + ) self.cache_store_package(obj) if created: if not self.options['quiet']: sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg)) - ' Set all versions dead, then set found versions alive and delete old versions ' + # Set all versions dead, then set found versions alive and + # delete old versions if not self.options['all']: - Version.objects.filter(package=obj, packaged=True).update(alive=False) + Version.objects.filter( + package=obj, + packaged=True + ).update(alive=False) return obj @@ -232,11 +256,15 @@ class Command(BaseCommand): overlay = 'gentoo' created = False - obj = self.cache_get_version(package.category, package.name, ver, rev, slot, overlay) + obj = self.cache_get_version( + package.category, package.name, ver, rev, slot, overlay + ) if not obj: - obj, created = Version.objects.get_or_create(package=package, slot=slot, - revision=rev, version=ver, - overlay=overlay) + obj, created = Version.objects.get_or_create( + package=package, slot=slot, + revision=rev, version=ver, + overlay=overlay + ) obj.alive = True obj.packaged = True @@ -245,7 +273,8 @@ class Command(BaseCommand): if created: self.cache_store_version(obj) - ''' nothing to do (note: it can't be an upstream version because overlay can't be empty here) ''' + # nothing to do (note: it can't be an upstream version because + # overlay can't be empty here) if not created: return @@ -287,8 +316,10 @@ class Command(BaseCommand): if self.options['no-log']: continue - entry = VersionLog.objects.create(package=version.package, - action=VersionLog.VERSION_REMOVED) + entry = VersionLog.objects.create( + package=version.package, + action=VersionLog.VERSION_REMOVED + ) entry.slot = version.slot entry.revision = version.revision entry.version = version.version @@ -296,4 +327,3 @@ class Command(BaseCommand): entry.save() Version.objects.filter(packaged=True, alive=False).delete() - diff --git a/euscanwww/djeuscan/management/commands/scan-upstream.py b/euscanwww/djeuscan/management/commands/scan-upstream.py index 1c79f3f..dd00619 100644 --- a/euscanwww/djeuscan/management/commands/scan-upstream.py +++ b/euscanwww/djeuscan/management/commands/scan-upstream.py @@ -1,19 +1,17 @@ import subprocess import portage import sys -import os import re from StringIO import StringIO -from datetime import datetime -from portage import versions from optparse import make_option from django.utils import timezone from django.db.transaction import commit_on_success -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from djeuscan.models import Package, Version, EuscanResult, VersionLog + class Command(BaseCommand): _overlays = {} @@ -60,7 +58,7 @@ class Command(BaseCommand): elif args: packages = list(args) else: - packages = [ package[:-1] for package in sys.stdin.readlines() ] + packages = [package[:-1] for package in sys.stdin.readlines()] self.scan(options, packages) @@ -83,8 +81,12 @@ class Command(BaseCommand): def parse_output(self, options, output): from portage.versions import _cp - package_re = re.compile(r'^ \* (?P' + _cp + ') \[(?P.*?)\]$') - version_re = re.compile(r'^Upstream Version: (?P.*?) (?P.*?)$') + package_re = re.compile( + r'^ \* (?P' + _cp + ') \[(?P.*?)\]$' + ) + version_re = re.compile( + r'^Upstream Version: (?P.*?) (?P.*?)$' + ) package = None log = "" @@ -124,7 +126,6 @@ class Command(BaseCommand): obj.datetime = timezone.now() obj.save() - def store_package(self, options, cpv): cat, pkg, ver, rev = portage.catpkgsplit(cpv) @@ -133,15 +134,16 @@ class Command(BaseCommand): if created and not options['quiet']: sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg)) - ' Set all versions dead, then set found versions alive and delete old versions ' + # Set all versions dead, then set found versions alive and + # delete old versions Version.objects.filter(package=obj, packaged=False).update(alive=False) return obj def store_version(self, options, package, ver, url): - obj, created = Version.objects.get_or_create(package=package, slot='', - revision='r0', version=ver, - overlay='') + obj, created = Version.objects.get_or_create( + package=package, slot='', revision='r0', version=ver, overlay='' + ) obj.alive = True obj.urls = url @@ -155,7 +157,10 @@ class Command(BaseCommand): if not options['quiet']: sys.stdout.write('+ [u] %s %s\n' % (obj, url)) - entry = VersionLog.objects.create(package=package, action=VersionLog.VERSION_ADDED) + entry = VersionLog.objects.create( + package=package, + action=VersionLog.VERSION_ADDED + ) entry.slot = '' entry.revision = 'r0' entry.version = ver @@ -165,12 +170,14 @@ class Command(BaseCommand): package.n_versions += 1 package.save() - @commit_on_success def purge_versions(self, options): ' For each dead versions ' for version in Version.objects.filter(packaged=False, alive=False): - entry = VersionLog.objects.create(package=version.package, action=VersionLog.VERSION_REMOVED) + entry = VersionLog.objects.create( + package=version.package, + action=VersionLog.VERSION_REMOVED + ) entry.slot = version.slot entry.revision = version.revision entry.version = version.version @@ -183,4 +190,3 @@ class Command(BaseCommand): if not options['quiet']: sys.stdout.write('- [u] %s %s\n' % (version, version.urls)) Version.objects.filter(packaged=False, alive=False).delete() - diff --git a/euscanwww/djeuscan/management/commands/update-counters.py b/euscanwww/djeuscan/management/commands/update-counters.py index fc29845..de3ea03 100644 --- a/euscanwww/djeuscan/management/commands/update-counters.py +++ b/euscanwww/djeuscan/management/commands/update-counters.py @@ -1,10 +1,7 @@ -import datetime - from optparse import make_option -from django.db.models import Count, Sum from django.db.transaction import commit_on_success -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from django.utils import timezone from djeuscan.models import Package, Herd, Maintainer, Version @@ -13,6 +10,7 @@ from djeuscan import charts from distutils.version import StrictVersion, LooseVersion + def compare_versions(version1, version2): try: return cmp(StrictVersion(version1), StrictVersion(version2)) @@ -20,6 +18,7 @@ def compare_versions(version1, version2): except ValueError: return cmp(LooseVersion(version1), LooseVersion(version2)) + class Command(BaseCommand): _overlays = {} help = 'Update counters' @@ -98,7 +97,8 @@ class Command(BaseCommand): return if version['version'].startswith('9999'): return - if compare_versions(storage[key]['version'], version['version']) < 0: + if compare_versions(storage[key]['version'], + version['version']) < 0: storage[key] = version if not options['fast']: @@ -125,25 +125,36 @@ class Command(BaseCommand): package.n_packaged = n_packaged.get(package.id, 0) package.n_overlay = n_overlay.get(package.id, 0) - default = {'id' : None} - package.last_version_gentoo_id = last_versions_gentoo.get(package.id, default)['id'] - package.last_version_overlay_id = last_versions_overlay.get(package.id, default)['id'] - package.last_version_upstream_id = last_versions_upstream.get(package.id, default)['id'] + default = {'id': None} + package.last_version_gentoo_id = last_versions_gentoo.get( + package.id, default + )['id'] + package.last_version_overlay_id = last_versions_overlay.get( + package.id, default + )['id'] + package.last_version_upstream_id = last_versions_upstream.get( + package.id, default + )['id'] package.save() n_packages_gentoo = int(package.n_packaged == package.n_versions) - n_packages_overlay = int(package.n_overlay and package.n_packaged + package.n_overlay == package.n_versions) - n_packages_outdated = int(package.n_packaged + package.n_overlay < package.n_versions) + n_packages_overlay = int(package.n_overlay and package.n_packaged \ + + package.n_overlay == package.n_versions) + n_packages_outdated = int(package.n_packaged + package.n_overlay \ + < package.n_versions) def update_row(storage, key): - storage[key].n_packages_gentoo += n_packages_gentoo - storage[key].n_packages_overlay += n_packages_overlay + storage[key].n_packages_gentoo += n_packages_gentoo + storage[key].n_packages_overlay += n_packages_overlay storage[key].n_packages_outdated += n_packages_outdated - storage[key].n_versions_gentoo += package.n_packaged - storage[key].n_versions_overlay += package.n_overlay - storage[key].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay + storage[key].n_versions_gentoo += package.n_packaged + storage[key].n_versions_overlay += package.n_overlay + storage[key].n_versions_upstream += package.n_versions - \ + package.n_packaged - \ + package.n_overlay + def update_log(storage, qs): for row in qs: update_row(storage, row['id']) @@ -153,13 +164,15 @@ class Command(BaseCommand): update_log(maintainers, package.maintainers.all().values('id')) update_row(categories, package.category) - wlog.n_packages_gentoo += n_packages_gentoo - wlog.n_packages_overlay += n_packages_overlay + wlog.n_packages_gentoo += n_packages_gentoo + wlog.n_packages_overlay += n_packages_overlay wlog.n_packages_outdated += n_packages_outdated - wlog.n_versions_gentoo += package.n_packaged - wlog.n_versions_overlay += package.n_overlay - wlog.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay + wlog.n_versions_gentoo += package.n_packaged + wlog.n_versions_overlay += package.n_overlay + wlog.n_versions_upstream += package.n_versions - \ + package.n_packaged - \ + package.n_overlay if options['nolog']: return diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py index 180f9d5..8555c2e 100644 --- a/euscanwww/djeuscan/models.py +++ b/euscanwww/djeuscan/models.py @@ -1,5 +1,6 @@ from django.db import models + class Herd(models.Model): herd = models.CharField(max_length=128, unique=True) email = models.CharField(max_length=128, blank=True, null=True) @@ -9,6 +10,7 @@ class Herd(models.Model): return '%s <%s>' % (self.herd, self.email) return self.herd + class Maintainer(models.Model): name = models.CharField(max_length=128) email = models.CharField(max_length=128, unique=True) @@ -16,6 +18,7 @@ class Maintainer(models.Model): def __unicode__(self): return '%s <%s>' % (self.name, self.email) + class Package(models.Model): category = models.CharField(max_length=128) name = models.CharField(max_length=128) @@ -30,15 +33,18 @@ class Package(models.Model): n_overlay = models.IntegerField(default=0) ' And we also pre-compute last versions ' - last_version_gentoo = models.ForeignKey('Version', blank=True, null=True, - related_name="last_version_gentoo", - on_delete=models.SET_NULL) - last_version_overlay = models.ForeignKey('Version', blank=True, null=True, - related_name="last_version_overlay", - on_delete=models.SET_NULL) - last_version_upstream = models.ForeignKey('Version', blank=True, null=True, - related_name="last_version_upstream", - on_delete=models.SET_NULL) + last_version_gentoo = models.ForeignKey( + 'Version', blank=True, null=True, related_name="last_version_gentoo", + on_delete=models.SET_NULL + ) + last_version_overlay = models.ForeignKey( + 'Version', blank=True, null=True, related_name="last_version_overlay", + on_delete=models.SET_NULL + ) + last_version_upstream = models.ForeignKey( + 'Version', blank=True, null=True, related_name="last_version_upstream", + on_delete=models.SET_NULL + ) def __unicode__(self): return '%s/%s' % (self.category, self.name) @@ -46,6 +52,7 @@ class Package(models.Model): class Meta: unique_together = ['category', 'name'] + class Version(models.Model): package = models.ForeignKey(Package) slot = models.CharField(max_length=128) @@ -57,13 +64,15 @@ class Version(models.Model): alive = models.BooleanField(default=True, db_index=True) def __unicode__(self): - return '%s/%s-%s-%s:%s [%s]' % (self.package.category, self.package.name, - self.version, self.revision, self.slot, - self.overlay) + return '%s/%s-%s-%s:%s [%s]' % ( + self.package.category, self.package.name, self.version, + self.revision, self.slot, self.overlay + ) class Meta: unique_together = ['package', 'slot', 'revision', 'version', 'overlay'] + class VersionLog(models.Model): VERSION_ADDED = 1 VERSION_REMOVED = 2 @@ -87,58 +96,67 @@ class VersionLog(models.Model): def __unicode__(self): txt = '+ ' if self.action == self.VERSION_ADDED else '- ' - txt += '%s/%s-%s-%s:%s [%s]' % (self.package.category, self.package.name, - self.version, self.revision, self.slot, - self.overlay if self.overlay else '') + txt += '%s/%s-%s-%s:%s [%s]' % ( + self.package.category, self.package.name, self.version, + self.revision, self.slot, + self.overlay if self.overlay else '' + ) return txt + class EuscanResult(models.Model): package = models.ForeignKey(Package) datetime = models.DateTimeField() result = models.TextField(blank=True) + # Keep data for charts class Log(models.Model): datetime = models.DateTimeField() ' Packages up to date in the main portage tree ' - n_packages_gentoo = models.IntegerField(default=0) + n_packages_gentoo = models.IntegerField(default=0) ' Packages up to date in an overlay ' - n_packages_overlay = models.IntegerField(default=0) + n_packages_overlay = models.IntegerField(default=0) ' Packages outdated ' n_packages_outdated = models.IntegerField(default=0) ' Versions in the main portage tree ' - n_versions_gentoo = models.IntegerField(default=0) + n_versions_gentoo = models.IntegerField(default=0) ' Versions in overlays ' - n_versions_overlay = models.IntegerField(default=0) + n_versions_overlay = models.IntegerField(default=0) ' Upstream versions, not in the main tree or overlays ' n_versions_upstream = models.IntegerField(default=0) def __unicode__(self): - return u'[%d:%d:%d] [%d:%d:%d]' % \ - (self.n_packages_gentoo, self.n_packages_overlay, self.n_packages_outdated, \ - self.n_versions_gentoo, self.n_versions_overlay, self.n_versions_upstream) + return u'[%d:%d:%d] [%d:%d:%d]' % ( + self.n_packages_gentoo, self.n_packages_overlay, + self.n_packages_outdated, self.n_versions_gentoo, + self.n_versions_overlay, self.n_versions_upstream + ) + class WorldLog(Log): def __unicode__(self): return u'world ' + Log.__unicode__(self) + class CategoryLog(Log): category = models.CharField(max_length=128) def __unicode__(self): return u'%s %s' % (self.category, Log.__unicode__(self)) + class HerdLog(Log): herd = models.ForeignKey(Herd) def __unicode__(self): return u'%s %s' % (self.herd, Log.__unicode__(self)) + class MaintainerLog(Log): maintainer = models.ForeignKey(Maintainer) def __unicode__(self): return u'%s %s' % (self.maintainer, Log.__unicode__(self)) - diff --git a/euscanwww/djeuscan/templatetags/div.py b/euscanwww/djeuscan/templatetags/div.py index 60b97f1..9acc319 100644 --- a/euscanwww/djeuscan/templatetags/div.py +++ b/euscanwww/djeuscan/templatetags/div.py @@ -2,7 +2,8 @@ from django import template register = template.Library() + def div(value, arg=None): - return float(value)/float(arg) + return float(value) / float(arg) register.filter('div', div) diff --git a/euscanwww/djeuscan/templatetags/math_.py b/euscanwww/djeuscan/templatetags/math_.py index 226ea8f..d9250b3 100644 --- a/euscanwww/djeuscan/templatetags/math_.py +++ b/euscanwww/djeuscan/templatetags/math_.py @@ -1,11 +1,14 @@ -from django.template import Node, Library +from django.template import Library import math register = Library() # taken from http://lybniz2.sourceforge.net/safeeval.html # make a list of safe functions -math_safe_list = ['acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'cosh', 'degrees', 'e', 'exp', 'fabs', 'floor', 'fmod', 'frexp', 'hypot', 'ldexp', 'log', 'log10', 'modf', 'pi', 'pow', 'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh'] +math_safe_list = ['acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'cosh', + 'degrees', 'e', 'exp', 'fabs', 'floor', 'fmod', 'frexp', + 'hypot', 'ldexp', 'log', 'log10', 'modf', 'pi', 'pow', + 'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh'] # use the list to filter the local namespace math_safe_dict = dict([(k, getattr(math, k)) for k in math_safe_list]) @@ -13,11 +16,13 @@ math_safe_dict = dict([(k, getattr(math, k)) for k in math_safe_list]) # add any needed builtins back in. math_safe_dict['abs'] = abs + @register.filter('math') def math_(lopr, expr): """Evals a math expression and returns it's value. - "$1" is a placeholder. Insert "$1" in the expression where the value is to be used. All math functions such as abs, sin, cos, floor are supported. + "$1" is a placeholder. Insert "$1" in the expression where the value is + to be used. All math functions such as abs, sin, cos, floor are supported. Example, a. You will be redirected in {{ seconds|math:"$1 / 60.0" }} minutes b. Square of {{ x }} is {{ x|math:"$1 * $1" }} @@ -25,5 +30,6 @@ def math_(lopr, expr): d. Given x = {{ x }}, (2 + x) * 6 = {{ x|math:"(2 + $1) * 6" }} """ if lopr: - return eval(expr.replace('$1', str(lopr)), {"__builtins__": None}, math_safe_dict) + return eval(expr.replace('$1', str(lopr)), {"__builtins__": None}, + math_safe_dict) return '' diff --git a/euscanwww/djeuscan/templatetags/mul.py b/euscanwww/djeuscan/templatetags/mul.py index c28f270..dbad300 100644 --- a/euscanwww/djeuscan/templatetags/mul.py +++ b/euscanwww/djeuscan/templatetags/mul.py @@ -2,7 +2,8 @@ from django import template register = template.Library() + def mul(value, arg=None): - return value*arg + return value * arg register.filter('mul', mul) diff --git a/euscanwww/djeuscan/templatetags/packages.py b/euscanwww/djeuscan/templatetags/packages.py index 520a1fc..370e4a5 100644 --- a/euscanwww/djeuscan/templatetags/packages.py +++ b/euscanwww/djeuscan/templatetags/packages.py @@ -2,16 +2,19 @@ from django import template register = template.Library() + @register.inclusion_tag('euscan/_packages.html', takes_context=True) def packages(context, packages): context['packages'] = packages return context + @register.inclusion_tag('euscan/_package_cols.html', takes_context=True) def package_cols(context, infos): context['infos'] = infos return context + @register.inclusion_tag('euscan/_package_bar.html', takes_context=True) def package_bar(context, infos): context['infos'] = infos diff --git a/euscanwww/djeuscan/templatetags/sub.py b/euscanwww/djeuscan/templatetags/sub.py index 9da87c1..47f52a1 100644 --- a/euscanwww/djeuscan/templatetags/sub.py +++ b/euscanwww/djeuscan/templatetags/sub.py @@ -2,7 +2,8 @@ from django import template register = template.Library() + def sub(value, arg=None): - return value-arg + return value - arg register.filter('sub', sub) diff --git a/euscanwww/djeuscan/templatetags/timedelta.py b/euscanwww/djeuscan/templatetags/timedelta.py index 6307575..e7656ae 100644 --- a/euscanwww/djeuscan/templatetags/timedelta.py +++ b/euscanwww/djeuscan/templatetags/timedelta.py @@ -6,6 +6,7 @@ from datetime import datetime register = template.Library() + def timedelta(value, arg=None): if not value: return '' @@ -16,8 +17,8 @@ def timedelta(value, arg=None): if settings.USE_TZ: cmp = make_aware(cmp, get_default_timezone()) if value > cmp: - return "in %s" % timesince(cmp,value) + return "in %s" % timesince(cmp, value) else: - return "%s ago" % timesince(value,cmp) + return "%s ago" % timesince(value, cmp) -register.filter('timedelta',timedelta) +register.filter('timedelta', timedelta) diff --git a/euscanwww/djeuscan/tests.py b/euscanwww/djeuscan/tests.py index 2247054..6b60485 100644 --- a/euscanwww/djeuscan/tests.py +++ b/euscanwww/djeuscan/tests.py @@ -7,6 +7,7 @@ Replace these with more appropriate tests for your application. from django.test import TestCase + class SimpleTest(TestCase): def test_basic_addition(self): """ @@ -20,4 +21,3 @@ Another way to test that 1 + 1 is equal to 2. >>> 1 + 1 == 2 True """} - diff --git a/euscanwww/djeuscan/urls.py b/euscanwww/djeuscan/urls.py index 5ce8b6d..74ba23e 100644 --- a/euscanwww/djeuscan/urls.py +++ b/euscanwww/djeuscan/urls.py @@ -1,29 +1,36 @@ -from django.conf.urls.defaults import * -from feeds import * +from django.conf.urls.defaults import url, patterns, include +from feeds import PackageFeed, CategoryFeed, HerdFeed, MaintainerFeed, \ + GlobalFeed package_patterns = patterns('djeuscan.views', - url(r'^(?P[\w+][\w+.-]*)/(?P[\w+][\w+.-]*)/feed/$', PackageFeed(), name='package_feed'), + url(r'^(?P[\w+][\w+.-]*)/(?P[\w+][\w+.-]*)/feed/$', + PackageFeed(), name='package_feed'), (r'^(?P[\w+][\w+.-]*)/(?P[\w+][\w+.-]*)/$', 'package'), ) categories_patterns = patterns('djeuscan.views', (r'^(?P[\w+][\w+.-]*)/(view/)?$', 'category'), - url(r'^(?P[\w+][\w+.-]*)/feed/$', CategoryFeed(), name='category_feed'), - (r'^(?P[\w+][\w+.-]*)/charts/(?P[\w\-]+).png$', 'chart_category'), + url(r'^(?P[\w+][\w+.-]*)/feed/$', CategoryFeed(), + name='category_feed'), + (r'^(?P[\w+][\w+.-]*)/charts/(?P[\w\-]+).png$', + 'chart_category'), (r'^$', 'categories'), ) herds_patterns = patterns('djeuscan.views', (r'^(?P[\@\{\}\w+.-]*)/(view/)?$', 'herd'), url(r'^(?P[\@\{\}\w+.-]*)/feed/$', HerdFeed(), name='herd_feed'), - (r'^(?P[\@\{\}\w+.-]*)/charts/(?P[\w\-]+).png$', 'chart_herd'), + (r'^(?P[\@\{\}\w+.-]*)/charts/(?P[\w\-]+).png$', + 'chart_herd'), (r'^$', 'herds'), ) maintainers_patterns = patterns('djeuscan.views', (r'^(?P\d+)/(view/)?$', 'maintainer'), - url(r'^(?P\d+)/feed/$', MaintainerFeed(), name='maintainer_feed'), - (r'^(?P\d+)/charts/(?P[\w\-]+).png$', 'chart_maintainer'), + url(r'^(?P\d+)/feed/$', MaintainerFeed(), + name='maintainer_feed'), + (r'^(?P\d+)/charts/(?P[\w\-]+).png$', + 'chart_maintainer'), (r'^$', 'maintainers'), ) diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py index 4d253f7..c5cc46a 100644 --- a/euscanwww/djeuscan/views.py +++ b/euscanwww/djeuscan/views.py @@ -1,5 +1,5 @@ from annoying.decorators import render_to -from django.http import HttpResponse, Http404 +from django.http import Http404 from django.shortcuts import get_object_or_404 from django.db.models import Sum, Max @@ -10,84 +10,117 @@ import charts """ Views """ + @render_to('euscan/index.html') def index(request): ctx = {} - ctx['n_packaged'] = charts.xint(Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum']) - ctx['n_overlay'] = charts.xint(Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum']) - ctx['n_versions'] = charts.xint(Package.objects.aggregate(Sum('n_versions'))['n_versions__sum']) - ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - ctx['n_overlay'] + ctx['n_packaged'] = charts.xint( + Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum'] + ) + ctx['n_overlay'] = charts.xint( + Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum'] + ) + ctx['n_versions'] = charts.xint( + Package.objects.aggregate(Sum('n_versions'))['n_versions__sum'] + ) + ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - \ + ctx['n_overlay'] ctx['n_packages'] = Package.objects.count() ctx['n_herds'] = Herd.objects.count() ctx['n_maintainers'] = Maintainer.objects.count() try: - ctx['last_scan'] = EuscanResult.objects.get(id=EuscanResult.objects.aggregate(Max('id'))['id__max']).datetime + ctx['last_scan'] = EuscanResult.objects.get( + id=EuscanResult.objects.aggregate(Max('id'))['id__max'] + ).datetime except EuscanResult.DoesNotExist: ctx['last_scan'] = None return ctx + @render_to('euscan/logs.html') def logs(request): return {} + @render_to('euscan/categories.html') def categories(request): - categories = Package.objects.values('category').annotate(n_packaged=Sum('n_packaged'), - n_overlay=Sum('n_overlay'), - n_versions=Sum('n_versions')) + categories = Package.objects.values('category').annotate( + n_packaged=Sum('n_packaged'), + n_overlay=Sum('n_overlay'), + n_versions=Sum('n_versions') + ) + + return {'categories': categories} - return { 'categories' : categories } @render_to('euscan/category.html') def category(request, category): packages = Package.objects.filter(category=category) - packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream') + packages = packages.select_related( + 'last_version_gentoo', 'last_version_overlay', 'last_version_upstream' + ) print dir(packages[0]) if not packages: raise Http404 - return { 'category' : category, 'packages' : packages } + return {'category': category, 'packages': packages} + @render_to('euscan/herds.html') def herds(request): - # FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of 'INNER JOIN' + # FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of + # 'INNER JOIN' herds = Package.objects.filter(herds__isnull=False) - herds = herds.values('herds__herd').annotate(n_packaged=Sum('n_packaged'), - n_overlay=Sum('n_overlay'), - n_versions=Sum('n_versions')) - return { 'herds' : herds } + herds = herds.values('herds__herd').annotate( + n_packaged=Sum('n_packaged'), + n_overlay=Sum('n_overlay'), + n_versions=Sum('n_versions')) + return {'herds': herds} + @render_to('euscan/herd.html') def herd(request, herd): herd = get_object_or_404(Herd, herd=herd) packages = Package.objects.filter(herds__id=herd.id) - packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream') - return { 'herd' : herd, 'packages' : packages } + packages = packages.select_related( + 'last_version_gentoo', 'last_version_overlay', 'last_version_upstream' + ) + return {'herd': herd, 'packages': packages} + @render_to('euscan/maintainers.html') def maintainers(request): maintainers = Package.objects.filter(maintainers__isnull=False) - maintainers = maintainers.values('maintainers__id', 'maintainers__name', 'maintainers__email') - maintainers = maintainers.annotate(n_packaged=Sum('n_packaged'), - n_overlay=Sum('n_overlay'), - n_versions=Sum('n_versions')) + maintainers = maintainers.values( + 'maintainers__id', 'maintainers__name', 'maintainers__email' + ) + maintainers = maintainers.annotate( + n_packaged=Sum('n_packaged'), + n_overlay=Sum('n_overlay'), + n_versions=Sum('n_versions') + ) + + return {'maintainers': maintainers} - return { 'maintainers' : maintainers } @render_to('euscan/maintainer.html') def maintainer(request, maintainer_id): maintainer = get_object_or_404(Maintainer, id=maintainer_id) packages = Package.objects.filter(maintainers__id=maintainer.id) - packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream') - return { 'maintainer' : maintainer, 'packages' : packages } + packages = packages.select_related( + 'last_version_gentoo', 'last_version_overlay', 'last_version_upstream' + ) + return {'maintainer': maintainer, 'packages': packages} + @render_to('euscan/overlays.html') def overlays(request): overlays = Package.objects.values('version__overlay') overlays = overlays.exclude(version__overlay='') overlays = overlays.distinct() - return { 'overlays' : overlays } + return {'overlays': overlays} + @render_to('euscan/overlay.html') def overlay(request, overlay): @@ -97,7 +130,8 @@ def overlay(request, overlay): packages = packages.filter(version__overlay=overlay).distinct() if not packages: raise Http404 - return { 'overlay' : overlay, 'packages' : packages } + return {'overlay': overlay, 'packages': packages} + @render_to('euscan/package.html') def package(request, category, package): @@ -120,19 +154,23 @@ def package(request, category, package): packaged = sorted(packaged, key=version_key) upstream = sorted(upstream, key=version_key) - log = EuscanResult.objects.filter(package=package).order_by('-datetime')[:1] + log = EuscanResult.objects.filter(package=package).\ + order_by('-datetime')[:1] log = log[0] if log else None vlog = VersionLog.objects.filter(package=package).order_by('-id') - return { 'package' : package, 'packaged' : packaged, - 'upstream' : upstream, 'log' : log, 'vlog' : vlog } + + return {'package': package, 'packaged': packaged, + 'upstream': upstream, 'log': log, 'vlog': vlog} + @render_to('euscan/world.html') def world(request): world_form = WorldForm() packages_form = PackagesForm() - return { 'world_form' : world_form, - 'packages_form' : packages_form } + return {'world_form': world_form, + 'packages_form': packages_form} + @render_to('euscan/world_scan.html') def world_scan(request): @@ -157,28 +195,34 @@ def world_scan(request): except: pass - return { 'packages' : packages } + return {'packages': packages} @render_to("euscan/about.html") def about(request): return {} + @render_to("euscan/api.html") def api(request): return {} + @render_to("euscan/statistics.html") def statistics(request): return {} + def chart(request, **kwargs): from django.views.static import serve chart = kwargs['chart'] if 'chart' in kwargs else None if 'maintainer_id' in kwargs: - kwargs['maintainer'] = get_object_or_404(Maintainer, id=kwargs['maintainer_id']) + kwargs['maintainer'] = get_object_or_404( + Maintainer, + id=kwargs['maintainer_id'] + ) if 'herd' in kwargs: kwargs['herd'] = get_object_or_404(Herd, herd=kwargs['herd']) @@ -202,11 +246,14 @@ def chart(request, **kwargs): return serve(request, path, document_root=charts.CHARTS_ROOT) + def chart_maintainer(request, **kwargs): return chart(request, **kwargs) + def chart_herd(request, **kwargs): return chart(request, **kwargs) + def chart_category(request, **kwargs): return chart(request, **kwargs) diff --git a/euscanwww/euscanwww/urls.py b/euscanwww/euscanwww/urls.py index 2f9fbb3..9472001 100644 --- a/euscanwww/euscanwww/urls.py +++ b/euscanwww/euscanwww/urls.py @@ -22,4 +22,3 @@ if settings.DEBUG: 'document_root': os.path.join(settings.EUSCAN_ROOT, 'htdocs'), }), ) - diff --git a/euscanwww/manage.py b/euscanwww/manage.py index 3d31e2b..ce915e9 100755 --- a/euscanwww/manage.py +++ b/euscanwww/manage.py @@ -1,5 +1,6 @@ #!/usr/bin/env python -import os, sys +import os +import sys if __name__ == "__main__": os.environ.setdefault("DJANGO_SETTINGS_MODULE", "euscanwww.settings") diff --git a/euscanwww/scripts/regen-seq.py b/euscanwww/scripts/regen-seq.py index 746804d..1002685 100644 --- a/euscanwww/scripts/regen-seq.py +++ b/euscanwww/scripts/regen-seq.py @@ -1,4 +1,5 @@ -import os, sys +import os +import sys sys.path.insert(0, '../') os.environ['DJANGO_SETTINGS_MODULE'] = 'euscanwww.settings' @@ -19,6 +20,6 @@ try: rel_name = seq_name.split("_id_seq")[0] to_update.append((seq_name, rel_name,)) for row in to_update: - c.execute(r"SELECT setval('%s', max(id)) FROM %s"%row) + c.execute(r"SELECT setval('%s', max(id)) FROM %s" % row) finally: c.close() diff --git a/euscanwww/templates/_base.html b/euscanwww/templates/_base.html index 22f9220..635aad3 100644 --- a/euscanwww/templates/_base.html +++ b/euscanwww/templates/_base.html @@ -15,7 +15,7 @@ @@ -24,44 +24,44 @@ diff --git a/euscanwww/templates/euscan/_packages.html b/euscanwww/templates/euscan/_packages.html index 68e5115..c39d11c 100644 --- a/euscanwww/templates/euscan/_packages.html +++ b/euscanwww/templates/euscan/_packages.html @@ -19,7 +19,7 @@ - {{ package.category }}/{{ package.name }} + {{ package.category }}/{{ package.name }} {% package_bar package %} diff --git a/euscanwww/templates/euscan/category.html b/euscanwww/templates/euscan/category.html index 6bd887a..66e034d 100644 --- a/euscanwww/templates/euscan/category.html +++ b/euscanwww/templates/euscan/category.html @@ -47,7 +47,7 @@
Packages
- + {% endblock %} diff --git a/euscanwww/templates/euscan/herd.html b/euscanwww/templates/euscan/herd.html index 38598fc..98e9951 100644 --- a/euscanwww/templates/euscan/herd.html +++ b/euscanwww/templates/euscan/herd.html @@ -45,7 +45,7 @@
Packages
- + {% endblock %} diff --git a/euscanwww/templates/euscan/maintainer.html b/euscanwww/templates/euscan/maintainer.html index a8803ab..dfa38b0 100644 --- a/euscanwww/templates/euscan/maintainer.html +++ b/euscanwww/templates/euscan/maintainer.html @@ -45,7 +45,7 @@
Packages
- + {% endblock %} diff --git a/euscanwww/templates/euscan/maintainers.html b/euscanwww/templates/euscan/maintainers.html index b0c833e..443950b 100644 --- a/euscanwww/templates/euscan/maintainers.html +++ b/euscanwww/templates/euscan/maintainers.html @@ -24,11 +24,11 @@ - {% if maintainer.maintainers__name != maintainer.maintainers__email %} - {{ maintainer.maintainers__name }} <{{ maintainer.maintainers__email }}> - {% else %} - {{ maintainer.maintainers__name }} - {% endif %} + {% if maintainer.maintainers__name != maintainer.maintainers__email %} + {{ maintainer.maintainers__name }} <{{ maintainer.maintainers__email }}> + {% else %} + {{ maintainer.maintainers__name }} + {% endif %} {% package_bar maintainer %} diff --git a/euscanwww/templates/euscan/package.html b/euscanwww/templates/euscan/package.html index eafab82..819f143 100644 --- a/euscanwww/templates/euscan/package.html +++ b/euscanwww/templates/euscan/package.html @@ -51,7 +51,7 @@ {% for maintainer in package.maintainers.all %} {% if maintainer.name != maintainer.email %} - {{ maintainer.name }} + {{ maintainer.name }} <{{ maintainer.email }}> {% else %} @@ -68,12 +68,12 @@
    {% for version in packaged %}
  • - {% if version.overlay == "gentoo" %} - gentoo - {% else %} - overlays - {% endif %} - {{ version.version }}-{{ version.revision }} :{{ version.slot }} [{{ version.overlay }}] + {% if version.overlay == "gentoo" %} + gentoo + {% else %} + overlays + {% endif %} + {{ version.version }}-{{ version.revision }} :{{ version.slot }} [{{ version.overlay }}]
    • {% endfor %}
@@ -85,8 +85,8 @@
    {% for version in upstream %}
  • - upstream - {{ version.version }} - {{ version.urls }} + upstream + {{ version.version }} - {{ version.urls }}
    • {% endfor %}
@@ -101,14 +101,16 @@ {% else %}
  • {% endif %} - {% if version.overlay == "gentoo" %} - gentoo - {% else %}{% if version.overlay %} - overlays - {% else %} - upstream - {% endif %}{% endif %} - {{ version }} - {{ version.datetime }} + {% if version.overlay == "gentoo" %} + gentoo + {% elif version.overlay %} + overlays + {% else %} + upstream + {% endif %} + + {{ version }} - {{ version.datetime }} +
    • {% endfor %} diff --git a/pym/euscan/__init__.py b/pym/euscan/__init__.py index cafe173..391a7a3 100644 --- a/pym/euscan/__init__.py +++ b/pym/euscan/__init__.py @@ -5,7 +5,6 @@ __version__ = "git" -import sys from portage.output import EOutput @@ -19,15 +18,16 @@ CONFIG = { 'brute-force-false-watermark': 50, 'scan-dir': True, 'oneshot': True, - 'user-agent' : 'escan (http://euscan.iksaif.net)', - 'skip-robots-txt' : False, - 'cache' : False + 'user-agent': 'escan (http://euscan.iksaif.net)', + 'skip-robots-txt': False, + 'cache': False } output = EOutput(CONFIG['quiet']) BLACKLIST_VERSIONS = [ - # Compatibility package for running binaries linked against a pre gcc 3.4 libstdc++, won't be updated + # Compatibility package for running binaries linked against a + # pre gcc 3.4 libstdc++, won't be updated '>=sys-libs/libstdc++-v3-3.4', ] @@ -39,21 +39,24 @@ BLACKLIST_PACKAGES = [ ] SCANDIR_BLACKLIST_URLS = [ - 'mirror://rubygems/(.*)', # Not browsable - 'mirror://gentoo/(.*)' # Directory too big + 'mirror://rubygems/(.*)', # Not browsable + 'mirror://gentoo/(.*)' # Directory too big ] BRUTEFORCE_BLACKLIST_PACKAGES = [ - 'net-zope/plonepopoll' # infinite loop any http://plone.org/products/plonepopoll/releases/*/plonepopoll-2-6-1.tgz link will work + # infinite loop any + # http://plone.org/products/plonepopoll/releases/*/plonepopoll-2-6-1.tgz + # link will work + 'net-zope/plonepopoll' ] BRUTEFORCE_BLACKLIST_URLS = [ - 'http://(.*)dockapps.org/download.php/id/(.*)', # infinite loop - 'http://hydra.nixos.org/build/(.*)', # infinite loop - 'http://www.rennings.net/gentoo/distfiles/(.*)', # Doesn't respect 404, infinite loop - 'http://art.gnome.org/download/(.*)', # Doesn't respect 404, infinite loop - 'http://barelysufficient.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop - 'http://olemarkus.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop + 'http://(.*)dockapps.org/download.php/id/(.*)', # infinite loop + 'http://hydra.nixos.org/build/(.*)', # infinite loop + 'http://www.rennings.net/gentoo/distfiles/(.*)', # Doesn't respect 404, infinite loop + 'http://art.gnome.org/download/(.*)', # Doesn't respect 404, infinite loop + 'http://barelysufficient.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop + 'http://olemarkus.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop ] ROBOTS_TXT_BLACKLIST_DOMAINS = [ diff --git a/pym/euscan/handlers/__init__.py b/pym/euscan/handlers/__init__.py index ed5db31..41236cc 100644 --- a/pym/euscan/handlers/__init__.py +++ b/pym/euscan/handlers/__init__.py @@ -1,6 +1,7 @@ from euscan.handlers import generic, php, pypi, rubygem, kde, cpan -handlers = [ kde, php, pypi, rubygem, cpan, generic ] +handlers = [kde, php, pypi, rubygem, cpan, generic] + def find_best_handler(cpv, url): for handler in handlers: @@ -8,12 +9,14 @@ def find_best_handler(cpv, url): return handler return None + def scan(cpv, url): handler = find_best_handler(cpv, url) if handler: return handler.scan(cpv, url) return [] + def brute_force(cpv, url): handler = find_best_handler(cpv, url) if handler: diff --git a/pym/euscan/handlers/cpan.py b/pym/euscan/handlers/cpan.py index 89c5554..1b5a111 100644 --- a/pym/euscan/handlers/cpan.py +++ b/pym/euscan/handlers/cpan.py @@ -8,9 +8,11 @@ import euscan _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*") + def can_handle(cpv, url): return url.startswith('mirror://cpan/') + def guess_package(cp, url): match = _cpan_package_name_re.search(url) @@ -27,6 +29,7 @@ def guess_package(cp, url): return pkg + def gentoo_mangle_version(up_pv): pv = "" @@ -39,17 +42,20 @@ def gentoo_mangle_version(up_pv): c = up_pv[i] pv += c digits += int(c.isdigit()) - if c == '.': digits = 0 + if c == '.': + digits = 0 else: pv = up_pv return helpers.gentoo_mangle_version(pv) + def cpan_trim_version(pv): pv = re.sub('^[a-zA-Z]+', '', pv) pv = re.sub('[a-zA-Z]$', '', pv) return pv + def cpan_mangle_version(pv): pos = pv.find('.') if pos < 0: @@ -59,6 +65,7 @@ def cpan_mangle_version(pv): up_pv = cpan_trim_version(up_pv) return up_pv + def cpan_vercmp(cp, a, b): try: return float(a) - float(b) @@ -68,6 +75,7 @@ def cpan_vercmp(cp, a, b): else: return 1 + def scan(cpv, url): cp, ver, rev = portage.pkgsplit(cpv) pkg = guess_package(cp, url) @@ -107,15 +115,20 @@ def scan(cpv, url): if helpers.version_filtered(cp, up_ver, up_pv, cpan_vercmp): continue - url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % \ - (version['cpanid'][0], version['cpanid'][0:1], version['cpanid'], version['archive']) + url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % ( + version['cpanid'][0], + version['cpanid'][0:1], + version['cpanid'], + version['archive'] + ) if url == orig_url: continue - ret.append(( url, pv )) + ret.append((url, pv)) return ret + def brute_force(cpv, url): return [] diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/generic.py index bcf330d..ceb854b 100644 --- a/pym/euscan/handlers/generic.py +++ b/pym/euscan/handlers/generic.py @@ -6,10 +6,12 @@ from BeautifulSoup import BeautifulSoup import portage -from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS +from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \ + BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS from euscan import helpers import euscan + def scan_html(data, url, pattern): soup = BeautifulSoup(data) results = [] @@ -28,6 +30,7 @@ def scan_html(data, url, pattern): return results + def scan_ftp(data, url, pattern): buf = StringIO.StringIO(data) results = [] @@ -40,6 +43,7 @@ def scan_ftp(data, url, pattern): return results + def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): if not steps: return [] @@ -91,6 +95,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): return versions + def scan(cpv, url): for bu in SCANDIR_BLACKLIST_URLS: if re.match(bu, url): @@ -107,13 +112,17 @@ def scan(cpv, url): if ver not in resolved_url: newver = helpers.version_change_end_sep(ver) if newver and newver in resolved_url: - euscan.output.einfo("Version: using %s instead of %s" % (newver, ver)) + euscan.output.einfo( + "Version: using %s instead of %s" % (newver, ver) + ) ver = newver template = helpers.template_from_url(resolved_url, ver) if '${' not in template: - euscan.output.einfo("Url doesn't seems to depend on version: %s not found in %s" - % (ver, resolved_url)) + euscan.output.einfo( + "Url doesn't seems to depend on version: %s not found in %s" % + (ver, resolved_url) + ) return [] else: euscan.output.einfo("Scanning: %s" % template) @@ -121,6 +130,7 @@ def scan(cpv, url): steps = helpers.generate_scan_paths(template) return scan_directory_recursive(cp, ver, rev, "", steps, url) + def brute_force(cpv, url): cp, ver, rev = portage.pkgsplit(cpv) @@ -155,8 +165,9 @@ def brute_force(cpv, url): template = helpers.template_from_url(url, ver) if '${PV}' not in template: - euscan.output.einfo("Url doesn't seems to depend on full version: %s not found in %s" - % (ver, url)) + euscan.output.einfo( + "Url doesn't seems to depend on full version: %s not found in %s" % + (ver, url)) return [] else: euscan.output.einfo("Brute forcing: %s" % template) @@ -187,11 +198,14 @@ def brute_force(cpv, url): result.append([url, version]) if len(result) > CONFIG['brute-force-false-watermark']: - euscan.output.einfo("Broken server detected ! Skipping brute force.") + euscan.output.einfo( + "Broken server detected ! Skipping brute force." + ) return [] if CONFIG["brute-force-recursive"]: - for v in helpers.gen_versions(list(components), CONFIG["brute-force"]): + for v in helpers.gen_versions(list(components), + CONFIG["brute-force"]): if v not in versions and tuple(v) not in done: versions.append(v) @@ -200,5 +214,6 @@ def brute_force(cpv, url): return result + def can_handle(cpv, url): return True diff --git a/pym/euscan/handlers/kde.py b/pym/euscan/handlers/kde.py index eb7fbd8..00947c0 100644 --- a/pym/euscan/handlers/kde.py +++ b/pym/euscan/handlers/kde.py @@ -1,10 +1,12 @@ from euscan.handlers import generic + def can_handle(cpv, url): if url.startswith('mirror://kde/'): return True return False + def clean_results(results): ret = [] @@ -15,6 +17,7 @@ def clean_results(results): return ret + def scan(cpv, url): results = generic.scan(cpv, url) @@ -24,6 +27,7 @@ def scan(cpv, url): return clean_results(results) + def brute_force(cpv, url): results = generic.brute_force(cpv, url) diff --git a/pym/euscan/handlers/php.py b/pym/euscan/handlers/php.py index 8919279..a4c7267 100644 --- a/pym/euscan/handlers/php.py +++ b/pym/euscan/handlers/php.py @@ -6,6 +6,7 @@ import xml.dom.minidom from euscan import helpers import euscan + def can_handle(cpv, url): if url.startswith('http://pear.php.net/get/'): return True @@ -13,6 +14,7 @@ def can_handle(cpv, url): return True return False + def guess_package_and_channel(cp, url): match = re.search('http://(.*)/get/(.*)-(.*).tgz', url) @@ -24,6 +26,7 @@ def guess_package_and_channel(cp, url): return pkg, host + def scan(cpv, url): cp, ver, rev = portage.pkgsplit(cpv) pkg, channel = guess_package_and_channel(cp, url) @@ -61,9 +64,10 @@ def scan(cpv, url): if url == orig_url: continue - ret.append(( url, pv )) + ret.append((url, pv)) return ret + def brute_force(cpv, url): return [] diff --git a/pym/euscan/handlers/pypi.py b/pym/euscan/handlers/pypi.py index e50038e..1e477b7 100644 --- a/pym/euscan/handlers/pypi.py +++ b/pym/euscan/handlers/pypi.py @@ -1,5 +1,4 @@ import xmlrpclib -import pprint import re import portage @@ -7,9 +6,11 @@ import portage from euscan import helpers import euscan + def can_handle(cpv, url): return url.startswith('mirror://pypi/') + def guess_package(cp, url): match = re.search('mirror://pypi/\w+/(.*)/.*', url) if match: @@ -19,10 +20,10 @@ def guess_package(cp, url): return pkg + def scan(cpv, url): 'http://wiki.python.org/moin/PyPiXmlRpc' - package = guess_package(cpv, url) euscan.output.einfo("Using PyPi XMLRPC: " + package) @@ -44,10 +45,11 @@ def scan(cpv, url): if helpers.version_filtered(cp, ver, pv): continue urls = client.release_urls(package, up_pv) - urls = " ".join([ infos['url'] for infos in urls ]) - ret.append(( urls, pv )) + urls = " ".join([infos['url'] for infos in urls]) + ret.append((urls, pv)) return ret + def brute_force(cpv, url): return [] diff --git a/pym/euscan/handlers/rubygem.py b/pym/euscan/handlers/rubygem.py index 17ca4dd..ec57a2a 100644 --- a/pym/euscan/handlers/rubygem.py +++ b/pym/euscan/handlers/rubygem.py @@ -6,9 +6,11 @@ import urllib2 from euscan import helpers import euscan + def can_handle(cpv, url): return url.startswith('mirror://rubygems/') + def guess_gem(cpv, url): match = re.search('mirror://rubygems/(.*).gem', url) if match: @@ -23,12 +25,14 @@ def guess_gem(cpv, url): return pkg + def scan(cpv, url): 'http://guides.rubygems.org/rubygems-org-api/#gemversion' gem = guess_gem(cpv, url) if not gem: - euscan.output.eerror("Can't guess gem name using %s and %s" % (cpv, url)) + euscan.output.eerror("Can't guess gem name using %s and %s" % \ + (cpv, url)) return [] url = 'http://rubygems.org/api/v1/versions/%s.json' % gem @@ -61,9 +65,10 @@ def scan(cpv, url): if helpers.version_filtered(cp, ver, pv): continue url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv) - ret.append(( url, pv )) + ret.append((url, pv)) return ret + def brute_force(cpv, url): return [] diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py index 67996c2..4c04a86 100644 --- a/pym/euscan/helpers.py +++ b/pym/euscan/helpers.py @@ -18,6 +18,7 @@ from portage import dep from euscan import CONFIG, BLACKLIST_VERSIONS, ROBOTS_TXT_BLACKLIST_DOMAINS import euscan + def htop_vercmp(a, b): def fixver(v): if v in ['0.11', '0.12', '0.13']: @@ -27,12 +28,13 @@ def htop_vercmp(a, b): return simple_vercmp(fixver(a), fixver(b)) VERSION_CMP_PACKAGE_QUIRKS = { - 'sys-process/htop' : htop_vercmp + 'sys-process/htop': htop_vercmp } _v_end = '((-|_)(pre|p|beta|b|alpha|a|rc|r)\d*)' _v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))' + # Stolen from g-pypi def gentoo_mangle_version(up_pv): """Convert PV to MY_PV if needed @@ -146,6 +148,7 @@ def gentoo_mangle_version(up_pv): return pv + def cast_int_components(version): for i, obj in enumerate(version): try: @@ -154,6 +157,7 @@ def cast_int_components(version): pass return version + def simple_vercmp(a, b): if a == b: return 0 @@ -173,11 +177,13 @@ def simple_vercmp(a, b): else: return 1 + def vercmp(package, a, b): if package in VERSION_CMP_PACKAGE_QUIRKS: return VERSION_CMP_PACKAGE_QUIRKS[package](a, b) return simple_vercmp(a, b) + def version_is_nightly(a, b): a = pkg_resources.parse_version(a) b = pkg_resources.parse_version(b) @@ -188,6 +194,7 @@ def version_is_nightly(a, b): return True return False + def version_blacklisted(cp, version): rule = None cpv = '%s-%s' % (cp, version) @@ -205,6 +212,7 @@ def version_blacklisted(cp, version): euscan.output.einfo("%s is blacklisted by rule %s" % (cpv, bv)) return rule is not None + def version_change_end_sep(version): match = re.match('.*' + _v_end, version) if not match: @@ -218,6 +226,7 @@ def version_change_end_sep(version): return None return version.replace(end, newend) + def version_filtered(cp, base, version, vercmp=vercmp): if vercmp(cp, base, version) >= 0: return True @@ -230,6 +239,7 @@ def version_filtered(cp, base, version, vercmp=vercmp): return False + def generate_templates_vars(version): ret = [] @@ -246,6 +256,7 @@ def generate_templates_vars(version): ret.reverse() return ret + def template_from_url(url, version): prefix, chunks = url.split('://') chunks = chunks.split('/') @@ -261,6 +272,7 @@ def template_from_url(url, version): return prefix + "://" + "/".join(chunks) + def url_from_template(url, version): components = split_version(version) @@ -270,6 +282,7 @@ def url_from_template(url, version): return url + # Stolen from distutils.LooseVersion # Used for brute force to increment the version def split_version(version): @@ -282,6 +295,7 @@ def split_version(version): pass return components + def join_version(components): version = "" for i in range(len(components)): @@ -292,6 +306,7 @@ def join_version(components): version += "." return version + def increment_version(components, level): n = len(components) @@ -307,6 +322,7 @@ def increment_version(components, level): return components + def gen_versions(components, level): n = len(components) depth = level @@ -325,6 +341,7 @@ def gen_versions(components, level): return versions + def timeout_for_url(url): if 'sourceforge' in url: timeout = 15 @@ -332,13 +349,16 @@ def timeout_for_url(url): timeout = 5 return timeout + class HeadRequest(urllib2.Request): def get_method(self): return "HEAD" + """ RobotParser cache """ rpcache = {} + def urlallowed(url): if CONFIG['skip-robots-txt']: return True @@ -359,7 +379,7 @@ def urlallowed(url): baseurl = '%s://%s' % (protocol, domain) robotsurl = urlparse.urljoin(baseurl, 'robots.txt') - if rpcache.has_key(baseurl): + if baseurl in rpcache: rp = rpcache[baseurl] else: from socket import setdefaulttimeout, getdefaulttimeout @@ -379,6 +399,7 @@ def urlallowed(url): return rp.can_fetch(CONFIG['user-agent'], url) if rp else False + def urlopen(url, timeout=None, verb="GET"): if not urlallowed(url): euscan.output.einfo("Url '%s' blocked by robots.txt" % url) @@ -410,6 +431,7 @@ def urlopen(url, timeout=None, verb="GET"): return opener.open(request, None, timeout) + def tryurl(fileurl, template): result = True @@ -429,13 +451,16 @@ def tryurl(fileurl, template): headers = fp.info() - if 'Content-disposition' in headers and basename not in headers['Content-disposition']: + if 'Content-disposition' in headers and \ + basename not in headers['Content-disposition']: result = None elif 'Content-Length' in headers and headers['Content-Length'] == '0': result = None - elif 'Content-Type' in headers and 'text/html' in headers['Content-Type']: + elif 'Content-Type' in headers and \ + 'text/html' in headers['Content-Type']: result = None - elif 'Content-Type' in headers and 'application/x-httpd-php' in headers['Content-Type']: + elif 'Content-Type' in headers and \ + 'application/x-httpd-php' in headers['Content-Type']: result = None elif fp.geturl() != fileurl: regex = regex_from_template(template) @@ -443,10 +468,10 @@ def tryurl(fileurl, template): basename2 = os.path.basename(fp.geturl()) # Redirect to another (earlier?) version - if basename != basename2 and (re.match(regex, fp.geturl()) or re.match(baseregex, basename2)): + if basename != basename2 and (re.match(regex, fp.geturl()) or \ + re.match(baseregex, basename2)): result = None - if result: result = (fp.geturl(), fp.info()) @@ -459,6 +484,7 @@ def tryurl(fileurl, template): return result + def regex_from_template(template): # Escape template = re.escape(template) @@ -483,6 +509,7 @@ def regex_from_template(template): template = template + r'/?$' return template + def basedir_from_template(template): idx = template.find('${') if idx == -1: @@ -494,6 +521,7 @@ def basedir_from_template(template): return template[0:idx] + def generate_scan_paths(url): prefix, chunks = url.split('://') chunks = chunks.split('/') @@ -511,6 +539,7 @@ def generate_scan_paths(url): return steps + def parse_mirror(uri): from random import shuffle @@ -526,7 +555,7 @@ def parse_mirror(uri): return None mirrorname = uri[9:eidx] - path = uri[eidx+1:] + path = uri[eidx + 1:] if mirrorname in mirrors: mirrors = mirrors[mirrorname] diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py index fddb4fc..d6be6cb 100644 --- a/pym/euscan/scan.py +++ b/pym/euscan/scan.py @@ -4,10 +4,8 @@ import sys import portage from portage.dbapi import porttree -from portage.output import white, yellow, turquoise, green, teal, red, EOutput import gentoolkit.pprinter as pp -from gentoolkit import errors from gentoolkit.query import Query from gentoolkit.eclean.search import (port_settings) @@ -17,6 +15,7 @@ from euscan import helpers import euscan + def filter_versions(cp, versions): filtered = {} @@ -32,7 +31,8 @@ def filter_versions(cp, versions): filtered[version] = url - return [ (cp, filtered[version], version) for version in filtered ] + return [(cp, filtered[version], version) for version in filtered] + def scan_upstream_urls(cpv, urls): versions = [] @@ -72,7 +72,9 @@ def scan_upstream(query): ) if not matches: - sys.stderr.write(pp.warn("No package matching '%s'" % pp.pkgquery(query))) + sys.stderr.write( + pp.warn("No package matching '%s'" % pp.pkgquery(query)) + ) return [] matches = sorted(matches) @@ -87,11 +89,15 @@ def scan_upstream(query): return [] if pkg.cp in BLACKLIST_PACKAGES: - sys.stderr.write(pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp))) + sys.stderr.write( + pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp)) + ) return [] if not CONFIG['quiet']: - pp.uprint(" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name()))) + pp.uprint( + " * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name())) + ) pp.uprint() ebuild_path = pkg.ebuild_path() @@ -104,8 +110,8 @@ def scan_upstream(query): cpv = pkg.cpv metadata = { - "EAPI" : port_settings["EAPI"], - "SRC_URI" : pkg.environment("SRC_URI", False), + "EAPI": port_settings["EAPI"], + "SRC_URI": pkg.environment("SRC_URI", False), } use = frozenset(port_settings["PORTAGE_USE"].split()) try: @@ -113,7 +119,9 @@ def scan_upstream(query): aalist = porttree._parse_uri_map(cpv, metadata) except Exception as e: sys.stderr.write(pp.warn("%s\n" % str(e))) - sys.stderr.write(pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv))) + sys.stderr.write( + pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv)) + ) return [] if "mirror" in portage.settings.features: diff --git a/setup.py b/setup.py index b08e46b..4423562 100755 --- a/setup.py +++ b/setup.py @@ -81,8 +81,8 @@ setup( 'https://github.com/iksaif/euscan/tarball/' + ('master' if __version__ == '9999' else ('euscan-%s' % __version__)) ), - install_requires=['Django==1.3.1', 'django-annoying', 'South', - 'django-piston', 'matplotlib'], + install_requires=['Django==1.4', 'django-annoying', 'South', + 'django-piston', 'matplotlib', 'BeautifulSoup'], package_dir={'': 'pym'}, packages=packages, package_data={},