src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Lovely day for PEP8 and pylint!

Browse Source
This commit is contained in:
volpino
2012-04-28 18:16:05 +02:00
parent 4e4f9643ac
commit 8c91855a58
37 changed files with 722 additions and 388 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
euscanwww/djeuscan/admin.py
View File

@ -1,11 +1,14 @@
from models import *
from models import Package, Version, VersionLog, EuscanResult, Log, WorldLog, \
CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer
from django.contrib import admin
admin.site.register(Herd)
admin.site.register(Maintainer)
class PackageAdmin(admin.ModelAdmin):
search_fields = ('category', 'name')
admin.site.register(Package, PackageAdmin)
admin.site.register(Version)

17
euscanwww/djeuscan/api/emitters.py
View File

@ -1,15 +1,16 @@
from piston.emitters import Emitter, XMLEmitter
from piston.emitters import XMLEmitter
class EuscanXMLEmitter(XMLEmitter):
_parent = []
_known_parents = {
'vlog' : 'version',
'herds' : 'herd',
'maintainers' : 'maintainer',
'packaged' : 'version',
'upstream' : 'version',
'packages' : 'package',
'categories' : 'category'
'vlog': 'version',
'herds': 'herd',
'maintainers': 'maintainer',
'packaged': 'version',
'upstream': 'version',
'packages': 'package',
'categories': 'category'
}
def _push_parent(self, parent):

87
euscanwww/djeuscan/api/handlers.py
View File

@ -1,24 +1,26 @@
from piston.handler import AnonymousBaseHandler, BaseHandler
from piston.utils import rc, HttpStatusCode
from piston.handler import AnonymousBaseHandler
from piston.utils import rc
from django.db.models import Sum, Max
from django.core.exceptions import ObjectDoesNotExist
from django.forms.models import model_to_dict
from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, VersionLog
from djeuscan.forms import WorldForm, PackagesForm
from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
VersionLog
# replace default XMLEmitter with ours
from piston.emitters import Emitter
from emitters import EuscanXMLEmitter
Emitter.register('xml', EuscanXMLEmitter, 'text/xml; charset=utf-8')
def xint(i):
try:
return int(i)
except:
return 0
def renameFields(vqs, fields):
ret = []
for n in vqs:
@ -29,6 +31,7 @@ def renameFields(vqs, fields):
ret.append(n)
return ret
class catch_and_return(object):
def __init__(self, err, response):
self.err = err
@ -42,6 +45,7 @@ class catch_and_return(object):
return self.response
return wrapper
# /api/1.0/
class RootHandler(AnonymousBaseHandler):
allowed_methods = ('GET',)
@ -49,52 +53,70 @@ class RootHandler(AnonymousBaseHandler):
def read(self, request):
return {'api-version': '1.0'}
# /api/1.0/statistics
class StatisticsHandler(AnonymousBaseHandler):
allowed_methods = ('GET',)
def read(self, request):
_aggr = Package.objects.aggregate
data = {}
data['n_packaged'] = xint(Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum'])
data['n_overlay'] = xint(Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum'])
data['n_versions'] = xint(Package.objects.aggregate(Sum('n_versions'))['n_versions__sum'])
data['n_upstream'] = data['n_versions'] - data['n_packaged'] - data['n_overlay']
data['n_packaged'] = xint(_aggr(Sum('n_packaged'))['n_packaged__sum'])
data['n_overlay'] = xint(_aggr(Sum('n_overlay'))['n_overlay__sum'])
data['n_versions'] = xint(_aggr(Sum('n_versions'))['n_versions__sum'])
data['n_upstream'] = data['n_versions'] - data['n_packaged'] - \
data['n_overlay']
data['n_packages'] = Package.objects.count()
data['n_herds'] = Herd.objects.count()
data['n_maintainers'] = Maintainer.objects.count()
data['last_scan'] = EuscanResult.objects.get(id=EuscanResult.objects.aggregate(Max('id'))['id__max']).datetime
data['last_scan'] = EuscanResult.objects.get(
id=EuscanResult.objects.aggregate(Max('id'))['id__max']
).datetime
return data
# /api/1.0/maintainers
class MaintainersHandler(AnonymousBaseHandler):
allowed_methods = ('GET',)
def read(self, request):
maintainers = Package.objects.filter(maintainers__isnull=False)
maintainers = maintainers.values('maintainers__id', 'maintainers__name', 'maintainers__email')
maintainers = maintainers.values(
'maintainers__id',
'maintainers__name',
'maintainers__email'
)
maintainers = maintainers.annotate(n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
maintainers = renameFields(maintainers, [('maintainers__id', 'id'),
('maintainers__name', 'name'),
('maintainers__email', 'email')])
return { 'maintainers' : maintainers }
maintainers = renameFields(
maintainers,
[('maintainers__id', 'id'),
('maintainers__name', 'name'),
('maintainers__email', 'email')]
)
return {'maintainers': maintainers}
# /api/1.0/herds
class HerdsHandler(AnonymousBaseHandler):
allowed_methods = ('GET',)
def read(self, request):
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of 'INNER JOIN'
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN'
# instead of 'INNER JOIN'
herds = Package.objects.filter(herds__isnull=False)
herds = herds.values('herds__herd').annotate(n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
herds = herds.values('herds__herd').annotate(
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')
)
herds = renameFields(herds, [('herds__herd', 'herd')])
return { 'herds' : herds }
return {'herds': herds}
# /api/1.0/categories
class CategoriesHandler(AnonymousBaseHandler):
@ -106,7 +128,8 @@ class CategoriesHandler(AnonymousBaseHandler):
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
return { 'categories' : categories }
return {'categories': categories}
# /api/1.0/packages/by-maintainer/
# /api/1.0/packages/by-category/
@ -125,17 +148,21 @@ class PackagesHandler(AnonymousBaseHandler):
if 'category' in kwargs:
packages = Package.objects.filter(category=kwargs['category'])
data = { 'category' : kwargs['category'] }
data = {'category': kwargs['category']}
elif 'herd' in kwargs:
herd = Herd.objects.get(herd=kwargs['herd'])
packages = Package.objects.filter(herds__id=herd.id)
data = { 'herd' : herd }
data = {'herd': herd}
elif 'maintainer_id' in kwargs:
maintainer = Maintainer.objects.get(id=kwargs['maintainer_id'])
packages = Package.objects.filter(maintainers__id=maintainer.id)
data = { 'maintainer' : maintainer }
data = {'maintainer': maintainer}
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
packages = packages.select_related(
'last_version_gentoo',
'last_version_overlay',
'last_version_upstream'
)
data['packages'] = packages
if not data:
@ -143,6 +170,7 @@ class PackagesHandler(AnonymousBaseHandler):
return data
# /api/1.0/package/
class PackageHandler(AnonymousBaseHandler):
allowed_methods = ('GET',)
@ -152,7 +180,8 @@ class PackageHandler(AnonymousBaseHandler):
package = Package.objects.get(category=category, name=package)
package.homepages = package.homepage.split(' ')
versions = Version.objects.filter(package=package)
log = EuscanResult.objects.filter(package=package).order_by('-datetime')[:1]
log = EuscanResult.objects.filter(package=package).\
order_by('-datetime')[:1]
log = log[0] if log else None
vlog = VersionLog.objects.filter(package=package).order_by('-id')
@ -166,7 +195,8 @@ class PackageHandler(AnonymousBaseHandler):
version_log = []
for v in vlog:
v = model_to_dict(v, ['version', 'revision', 'slot', 'overlay', 'datetime', 'action'])
v = model_to_dict(v, ['version', 'revision', 'slot', 'overlay',
'datetime', 'action'])
if v['action'] == VersionLog.VERSION_ADDED:
v['action'] = 'added'
if v['action'] == VersionLog.VERSION_REMOVED:
@ -177,7 +207,10 @@ class PackageHandler(AnonymousBaseHandler):
packaged = []
for version in versions:
unpackaged = not version.packaged
version = model_to_dict(version, ['version', 'revision', 'slot', 'overlay', 'urls'])
version = model_to_dict(
version,
['version', 'revision', 'slot', 'overlay', 'urls']
)
if unpackaged:
upstream.append(version)
else:

32
euscanwww/djeuscan/api/urls.py
View File

@ -1,7 +1,8 @@
from django.conf.urls.defaults import *
from django.conf.urls.defaults import url, patterns
from piston.resource import Resource
from handlers import *
from handlers import RootHandler, StatisticsHandler, HerdsHandler, \
CategoriesHandler, MaintainersHandler, PackagesHandler, PackageHandler
root_handler = Resource(handler=RootHandler)
statistics_handler = Resource(handler=StatisticsHandler)
@ -12,16 +13,25 @@ packages_handler = Resource(handler=PackagesHandler)
package_handler = Resource(handler=PackageHandler)
urlpatterns = patterns('djeuscan.api.views',
url(r'^1.0/statistics\.(?P<emitter_format>.+)$', statistics_handler, name='api.views.statistics'),
url(r'^1.0/herds\.(?P<emitter_format>.+)$', herds_handler, name='api.views.herds'),
url(r'^1.0/categories\.(?P<emitter_format>.+)$', categories_handler, name='api.views.categories'),
url(r'^1.0/maintainers\.(?P<emitter_format>.+)$', maintainers_handler, name='api.views.maintainers'),
url(r'^1.0/statistics\.(?P<emitter_format>.+)$', statistics_handler,
name='api.views.statistics'),
url(r'^1.0/herds\.(?P<emitter_format>.+)$', herds_handler,
name='api.views.herds'),
url(r'^1.0/categories\.(?P<emitter_format>.+)$', categories_handler,
name='api.views.categories'),
url(r'^1.0/maintainers\.(?P<emitter_format>.+)$', maintainers_handler,
name='api.views.maintainers'),
url(r'^1.0/packages/by-maintainer/(?P<maintainer_id>\d+)\.(?P<emitter_format>.+)$', packages_handler, name='api.views.packages'),
url(r'^1.0/packages/by-herd/(?P<herd>[\@\{\}\w+.-]*)\.(?P<emitter_format>.+)?$', packages_handler, name='api.views.packages'),
url(r'^1.0/packages/by-category/(?P<category>[\w+][\w+.-]*)\.(?P<emitter_format>.+)?$', packages_handler, name='api.views.packages'),
url(r'^1.0/packages/by-maintainer/(?P<maintainer_id>\d+)\.(?P<emitter_format>.+)$',
packages_handler, name='api.views.packages'),
url(r'^1.0/packages/by-herd/(?P<herd>[\@\{\}\w+.-]*)\.(?P<emitter_format>.+)?$',
packages_handler, name='api.views.packages'),
url(r'^1.0/packages/by-category/(?P<category>[\w+][\w+.-]*)\.(?P<emitter_format>.+)?$',
packages_handler, name='api.views.packages'),
url(r'^1.0/package/(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)\.(?P<emitter_format>.+)$', package_handler, name='api.views.package'),
url(r'^1.0/package/(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)\.(?P<emitter_format>.+)$',
package_handler, name='api.views.package'),
url(r'^1.0/api\.(?P<emitter_format>.+)$', root_handler, name='api.views.root'),
url(r'^1.0/api\.(?P<emitter_format>.+)$',
root_handler, name='api.views.root'),
)

147
euscanwww/djeuscan/charts.py
View File

@ -3,14 +3,12 @@ import time
from euscanwww import settings
from django.db.models import F, Sum, Max
from djeuscan.models import Version, Package, Herd, Maintainer
from djeuscan.models import CategoryLog
from django.db.models import F, Sum
from djeuscan.models import Package
import rrdtool
import pylab
import matplotlib
CHARTS_ROOT = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts')
CHARTS_URL = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts')
@ -21,12 +19,14 @@ pylab.rcParams['axes.titlesize'] = 10.0
pylab.rcParams['xtick.labelsize'] = 8.0
pylab.rcParams['legend.fontsize'] = 8.0
def xint(i):
try:
return int(i)
except:
return 0
def chart_alive(name):
path = os.path.join(CHARTS_ROOT, name)
if not os.path.exists(path):
@ -37,6 +37,7 @@ def chart_alive(name):
return False
return True
def rrd_name(**kwargs):
name = ""
@ -51,6 +52,7 @@ def rrd_name(**kwargs):
return name
def chart_name(name, **kwargs):
name = name.replace('_', '-')
@ -67,6 +69,7 @@ def chart_name(name, **kwargs):
return name + ".png"
def getpackages(**kwargs):
packages = Package.objects
@ -79,6 +82,7 @@ def getpackages(**kwargs):
return packages
def cached_pylab_chart(f):
def new_f(*args, **kwds):
name = chart_name(f.func_name, **kwds)
@ -93,14 +97,16 @@ def cached_pylab_chart(f):
new_f.func_name = f.func_name
return new_f
@cached_pylab_chart
def pie_versions(**kwargs):
n_packaged = xint(getpackages(**kwargs).aggregate(Sum('n_packaged'))['n_packaged__sum'])
n_overlay = xint(getpackages(**kwargs).aggregate(Sum('n_overlay'))['n_overlay__sum'])
n_versions = xint(getpackages(**kwargs).aggregate(Sum('n_versions'))['n_versions__sum'])
gpk = getpackages(**kwargs)
n_packaged = xint(gpk.aggregate(Sum('n_packaged'))['n_packaged__sum'])
n_overlay = xint(gpk.aggregate(Sum('n_overlay'))['n_overlay__sum'])
n_versions = xint(gpk.aggregate(Sum('n_versions'))['n_versions__sum'])
n_upstream = n_versions - n_packaged - n_overlay
pylab.figure(1, figsize=(3.5,3.5))
pylab.figure(1, figsize=(3.5, 3.5))
if n_overlay:
labels = 'Gentoo', 'Overlays', 'Upstream'
@ -111,35 +117,43 @@ def pie_versions(**kwargs):
fracs = [n_packaged, n_upstream]
colors = '#008000', '#FF0000'
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', shadow=True)
pylab.title('Versions', bbox={'facecolor':'0.8', 'pad':5})
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%',
shadow=True)
pylab.title('Versions', bbox={'facecolor': '0.8', 'pad': 5})
@cached_pylab_chart
def pie_packages(**kwargs):
n_packages = getpackages(**kwargs).count()
n_packages_uptodate_main = getpackages(**kwargs).filter(n_versions=F('n_packaged')).count()
n_packages_uptodate_all = getpackages(**kwargs).filter(n_versions=F('n_packaged') + F('n_overlay')).count()
gpk = getpackages(**kwargs)
n_packages = gpk.count()
n_packages_uptodate_main = gpk.filter(n_versions=F('n_packaged')).count()
n_packages_uptodate_all = gpk.filter(n_versions=F('n_packaged') + \
F('n_overlay')).count()
n_packages_outdated = n_packages - n_packages_uptodate_all
n_packages_uptodate_ovl = n_packages_uptodate_all - n_packages_uptodate_main
n_packages_uptodate_ovl = n_packages_uptodate_all - \
n_packages_uptodate_main
pylab.figure(1, figsize=(3.5,3.5))
pylab.figure(1, figsize=(3.5, 3.5))
if n_packages_uptodate_ovl:
labels = 'Ok (gentoo)', 'Ok (overlays)', 'Outdated'
fracs = [n_packages_uptodate_main, n_packages_uptodate_ovl, n_packages_outdated]
fracs = [n_packages_uptodate_main, n_packages_uptodate_ovl,
n_packages_outdated]
colors = '#008000', '#0B17FD', '#FF0000'
else:
labels = 'Ok (gentoo)', 'Outdated'
fracs = [n_packages_uptodate_main, n_packages_outdated]
colors = '#008000', '#FF0000'
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', shadow=True)
pylab.title('Packages', bbox={'facecolor':'0.8', 'pad':5})
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%',
shadow=True)
pylab.title('Packages', bbox={'facecolor': '0.8', 'pad': 5})
def rrd_path(name):
return str(os.path.join(settings.RRD_ROOT, name + '.rrd'))
def rrd_create(name, start):
path = rrd_path(name)
if os.path.exists(path):
@ -156,13 +170,17 @@ def rrd_create(name, start):
'RRA:AVERAGE:0.5:5:200',
'RRA:AVERAGE:0.5:10:200')
def rrd_update(name, datetime, values):
now = time.mktime(datetime.timetuple())
rrd_create(name, now)
rrdtool.update(rrd_path(name),
'%d:%d:%d:%d:%d:%d:%d' % \
(now, values.n_packages_gentoo, values.n_packages_overlay, values.n_packages_outdated, \
values.n_versions_gentoo, values.n_versions_overlay, values.n_versions_upstream))
rrdtool.update(
rrd_path(name),
'%d:%d:%d:%d:%d:%d:%d' % \
(now, values.n_packages_gentoo, values.n_packages_overlay,
values.n_packages_outdated, values.n_versions_gentoo,
values.n_versions_overlay, values.n_versions_upstream)
)
"""
@ -175,6 +193,7 @@ def rrd_update(name, datetime, values):
[-J|--alt-autoscale-min]
"""
def cached_rrd_chart(f):
def new_f(*args, **kwds):
if 'period' not in kwds:
@ -215,47 +234,55 @@ def cached_rrd_chart(f):
new_f.func_name = f.func_name
return new_f
@cached_rrd_chart
def packages(**kwargs):
rrdtool.graph(str(kwargs['path']),
'--imgformat', 'PNG',
'--width', kwargs['width'],
'--height', kwargs['height'],
kwargs['graph-mode'],
'--color', 'CANVAS#FFFFFF00',
'--color', 'BACK#FFFFFF00',
rrdtool.graph(
str(kwargs['path']),
'--imgformat', 'PNG',
'--width', kwargs['width'],
'--height', kwargs['height'],
kwargs['graph-mode'],
'--color', 'CANVAS#FFFFFF00',
'--color', 'BACK#FFFFFF00',
'--start', kwargs['start'],
'--end', kwargs['end'],
'--vertical-label', kwargs['vertical-label'],
'--title', kwargs['title'],
'--lower-limit', '0',
'DEF:n_packages_gentoo=%s:n_packages_gentoo:AVERAGE' % (kwargs['rrd']),
'DEF:n_packages_overlay=%s:n_packages_overlay:AVERAGE' % \
(kwargs['rrd']),
'DEF:n_packages_outdated=%s:n_packages_outdated:AVERAGE' % \
(kwargs['rrd']),
'LINE1.25:n_packages_gentoo#008000:Gentoo',
'LINE1.25:n_packages_overlay#0B17FD:Overlay',
'LINE1.25:n_packages_outdated#FF0000:Outdated'
)
'--start', kwargs['start'],
'--end', kwargs['end'],
'--vertical-label', kwargs['vertical-label'],
'--title', kwargs['title'],
'--lower-limit', '0',
'DEF:n_packages_gentoo=%s:n_packages_gentoo:AVERAGE' % (kwargs['rrd']),
'DEF:n_packages_overlay=%s:n_packages_overlay:AVERAGE' % (kwargs['rrd']),
'DEF:n_packages_outdated=%s:n_packages_outdated:AVERAGE' % (kwargs['rrd']),
'LINE1.25:n_packages_gentoo#008000:Gentoo',
'LINE1.25:n_packages_overlay#0B17FD:Overlay',
'LINE1.25:n_packages_outdated#FF0000:Outdated')
@cached_rrd_chart
def versions(**kwargs):
rrdtool.graph(str(kwargs['path']),
'--imgformat', 'PNG',
'--width', kwargs['width'],
'--height', kwargs['height'],
kwargs['graph-mode'],
'--color', 'CANVAS#FFFFFF00',
'--color', 'BACK#FFFFFF00',
'--start', kwargs['start'],
'--end', kwargs['end'],
'--vertical-label', kwargs['vertical-label'],
'--title', kwargs['title'],
'--lower-limit', '0',
'DEF:n_versions_gentoo=%s:n_versions_gentoo:AVERAGE' % (kwargs['rrd']),
'DEF:n_versions_overlay=%s:n_versions_overlay:AVERAGE' % (kwargs['rrd']),
'DEF:n_versions_outdated=%s:n_versions_upstream:AVERAGE' % (kwargs['rrd']),
'LINE1.25:n_versions_gentoo#008000:Gentoo',
'LINE1.25:n_versions_overlay#0B17FD:Overlay',
'LINE1.25:n_versions_outdated#FF0000:Outdated')
rrdtool.graph(
str(kwargs['path']),
'--imgformat', 'PNG',
'--width', kwargs['width'],
'--height', kwargs['height'],
kwargs['graph-mode'],
'--color', 'CANVAS#FFFFFF00',
'--color', 'BACK#FFFFFF00',
'--start', kwargs['start'],
'--end', kwargs['end'],
'--vertical-label', kwargs['vertical-label'],
'--title', kwargs['title'],
'--lower-limit', '0',
'DEF:n_versions_gentoo=%s:n_versions_gentoo:AVERAGE' % (kwargs['rrd']),
'DEF:n_versions_overlay=%s:n_versions_overlay:AVERAGE' % \
(kwargs['rrd']),
'DEF:n_versions_outdated=%s:n_versions_upstream:AVERAGE' % \
(kwargs['rrd']),
'LINE1.25:n_versions_gentoo#008000:Gentoo',
'LINE1.25:n_versions_overlay#0B17FD:Overlay',
'LINE1.25:n_versions_outdated#FF0000:Outdated'
)

32
euscanwww/djeuscan/feeds.py
View File

@ -1,11 +1,10 @@
from django.contrib.syndication.views import Feed, FeedDoesNotExist
from django.shortcuts import get_object_or_404
from django.http import Http404
from django.utils.feedgenerator import Atom1Feed
from django.core.urlresolvers import reverse
from djeuscan.models import Version, Package, Herd, Maintainer, VersionLog
from djeuscan.views import *
from djeuscan.models import Package, Herd, Maintainer, VersionLog
class BaseFeed(Feed):
feed_type = Atom1Feed
@ -18,8 +17,8 @@ class BaseFeed(Feed):
def item_description(self, vlog):
if vlog.overlay:
txt = 'Version %s-%s [%s] of package %s ' % (vlog.version, vlog.revision,
vlog.slot, vlog.package)
txt = 'Version %s-%s [%s] of package %s ' % \
(vlog.version, vlog.revision, vlog.slot, vlog.package)
else:
txt = 'Version %s of package %s ' % (vlog.version, vlog.package)
if vlog.action == vlog.VERSION_REMOVED:
@ -36,8 +35,10 @@ class BaseFeed(Feed):
return txt
def item_link(self, vlog):
kwargs = {'category' : vlog.package.category, 'package' : vlog.package.name }
return reverse('djeuscan.views.package', kwargs=kwargs) + '#' + vlog.tag()
kwargs = {'category': vlog.package.category,
'package': vlog.package.name}
return "%s#%s" % (reverse('djeuscan.views.package', kwargs=kwargs),
vlog.tag())
def item_pubdate(self, vlog):
return vlog.datetime
@ -45,18 +46,20 @@ class BaseFeed(Feed):
def item_categories(self, vlog):
return [vlog.package.category]
class GlobalFeed(BaseFeed):
title = "euscan"
link = "/"
description = "Last euscan changes"
def categories(self):
categories = Package.objects.values('category').distinct();
return [ category['category'] for category in categories ]
categories = Package.objects.values('category').distinct()
return [category['category'] for category in categories]
def items(self):
return VersionLog.objects.order_by('-id')[:250]
class PackageFeed(BaseFeed):
feed_type = Atom1Feed
@ -67,7 +70,8 @@ class PackageFeed(BaseFeed):
return "%s" % package
def link(self, package):
return reverse('djeuscan.views.package', args=[package.category, package.name])
return reverse('djeuscan.views.package', args=[package.category,
package.name])
def description(self, package):
return package.description
@ -78,6 +82,7 @@ class PackageFeed(BaseFeed):
def item_description(self, vlog):
return ''
class MaintainerFeed(BaseFeed):
feed_type = Atom1Feed
@ -91,12 +96,14 @@ class MaintainerFeed(BaseFeed):
return "Last changes for %s" % maintainer
def link(self, maintainer):
return reverse('djeuscan.views.maintainer', kwargs={'maintainer_id' : maintainer.id})
return reverse('djeuscan.views.maintainer',
kwargs={'maintainer_id': maintainer.id})
def items(self, maintainer):
q = VersionLog.objects.filter(package__maintainers__id=maintainer.id)
return q.order_by('-id')[:50]
class HerdFeed(BaseFeed):
feed_type = Atom1Feed
@ -110,12 +117,13 @@ class HerdFeed(BaseFeed):
return "Last changes for %s" % herd
def link(self, herd):
return reverse('djeuscan.views.herd', kwargs={'herd' : herd.herd})
return reverse('djeuscan.views.herd', kwargs={'herd': herd.herd})
def items(self, herd):
q = VersionLog.objects.filter(package__herds__id=herd.id)
return q.order_by('-id')[:100]
class CategoryFeed(BaseFeed):
feed_type = Atom1Feed

2
euscanwww/djeuscan/forms.py
View File

@ -1,7 +1,9 @@
from django import forms
class WorldForm(forms.Form):
world = forms.FileField()
class PackagesForm(forms.Form):
packages = forms.CharField(widget=forms.Textarea)

3
euscanwww/djeuscan/management/commands/list-packages.py
View File

@ -1,8 +1,9 @@
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import Package
class Command(BaseCommand):
_overlays = {}
help = 'List packages'

13
euscanwww/djeuscan/management/commands/regen-rrds.py
View File

@ -1,11 +1,8 @@
import datetime
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from djeuscan import charts
class Command(BaseCommand):
_overlays = {}
help = 'Regenerate rrd database'
@ -15,10 +12,12 @@ class Command(BaseCommand):
charts.rrd_update('world', wlog.datetime, wlog)
for clog in CategoryLog.objects.all():
charts.rrd_update('category-%s' % clog.category, clog.datetime, clog)
charts.rrd_update('category-%s' % clog.category,
clog.datetime, clog)
for hlog in HerdLog.objects.all():
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
for mlog in MaintainerLog.objects.all():
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, mlog.datetime, mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
mlog.datetime, mlog)

53
euscanwww/djeuscan/management/commands/scan-metadata.py
View File

@ -1,19 +1,15 @@
import subprocess
import portage
import sys
import os
import re
from portage import versions
from optparse import make_option
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import Package, Herd, Maintainer
from gentoolkit.query import Query
from gentoolkit.errors import GentoolkitFatalError
class Command(BaseCommand):
_overlays = {}
@ -53,16 +49,20 @@ class Command(BaseCommand):
)
if not matches:
sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % query))
sys.stderr.write(
self.style.ERROR("Unknown package '%s'\n" % query)
)
return
matches = sorted(matches)
matches = sorted(matches)
pkg = matches.pop()
if '9999' in pkg.version and len(matches):
if '9999' in pkg.version and len(matches):
pkg = matches.pop()
if not obj:
obj, created = Package.objects.get_or_create(category=pkg.category, name=pkg.name)
obj, created = Package.objects.get_or_create(
category=pkg.category, name=pkg.name
)
else:
created = False
@ -70,22 +70,34 @@ class Command(BaseCommand):
obj.homepage = pkg.environment("HOMEPAGE")
obj.description = pkg.environment("DESCRIPTION")
except GentoolkitFatalError, err:
sys.stderr.write(self.style.ERROR("Gentoolkit fatal error: '%s'\n" % str(err)))
sys.stderr.write(
self.style.ERROR(
"Gentoolkit fatal error: '%s'\n" % str(err)
)
)
if created and not self.options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
if pkg.metadata:
herds = dict([(herd[0], herd) for herd in pkg.metadata.herds(True)])
maintainers = dict([(m.email, m) for m in pkg.metadata.maintainers()])
herds = dict(
[(herd[0], herd) for herd in pkg.metadata.herds(True)]
)
maintainers = dict(
[(m.email, m) for m in pkg.metadata.maintainers()]
)
existing_herds = [h.herd for h in obj.herds.all()]
new_herds = set(herds.keys()).difference(existing_herds)
old_herds = set(existing_herds).difference(herds.keys())
existing_maintainers = [m.email for m in obj.maintainers.all()]
new_maintainers = set(maintainers.keys()).difference(existing_maintainers)
old_maintainers = set(existing_maintainers).difference(maintainers.keys())
new_maintainers = set(
maintainers.keys()).difference(existing_maintainers
)
old_maintainers = set(
existing_maintainers).difference(maintainers.keys()
)
for herd in obj.herds.all():
if herd.herd in old_herds:
@ -101,7 +113,9 @@ class Command(BaseCommand):
for maintainer in new_maintainers:
maintainer = maintainers[maintainer]
maintainer = self.store_maintainer(maintainer.name, maintainer.email)
maintainer = self.store_maintainer(
maintainer.name, maintainer.email
)
obj.maintainers.add(maintainer)
obj.save()
@ -131,9 +145,12 @@ class Command(BaseCommand):
if created:
if not self.options['quiet']:
sys.stdout.write('+ [m] %s <%s>\n' % (name.encode('utf-8'), email))
sys.stdout.write(
'+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
)
if not maintainer.name or name not in [maintainer.name, email, '{nil}']:
if not maintainer.name or \
name not in [maintainer.name, email, '{nil}']:
maintainer.name = name
maintainer.save()

94
euscanwww/djeuscan/management/commands/scan-portage.py
View File

@ -4,13 +4,13 @@ import sys
import os
import re
from portage import versions
from optparse import make_option
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import Package, Version, VersionLog
class Command(BaseCommand):
_overlays = {}
@ -39,7 +39,8 @@ class Command(BaseCommand):
action='store_true',
dest='prefetch',
default=False,
help='Prefetch all versions and packages from DB to speedup full scan process.'),
help=('Prefetch all versions and packages from DB to '
'speedup full scan process.')),
make_option('--quiet',
action='store_true',
dest='quiet',
@ -49,7 +50,7 @@ class Command(BaseCommand):
args = '[package package ...]'
help = 'Scans portage tree and fills database'
_cache = {'packages' : {}, 'versions' : {}}
_cache = {'packages': {}, 'versions': {}}
def cache_hash_package(self, category, name):
return '%s/%s' % (category, name)
@ -59,22 +60,28 @@ class Command(BaseCommand):
self._cache['packages'][key] = package
def cache_get_package(self, category, name):
return self._cache['packages'].get(self.cache_hash_package(category, name))
return self._cache['packages'].get(
self.cache_hash_package(category, name)
)
def cache_hash_version(self, category, name, version, revision, slot, overlay):
def cache_hash_version(self, category, name, version, revision, slot,
overlay):
key = '%s/%s-%s-r%s %s %s' % (category, name,
version, revision,
slot, overlay)
return key
def cache_get_version(self, category, name, version, revision, slot, overlay):
key = self.cache_hash_version(category, name, version, revision, slot, overlay)
def cache_get_version(self, category, name, version, revision, slot,
overlay):
key = self.cache_hash_version(category, name, version, revision, slot,
overlay)
return self._cache['versions'].get(key)
def cache_store_version(self, version):
key = self.cache_hash_version(version.package.category, version.package.name,
version.version, version.revision, version.slot,
version.overlay)
key = self.cache_hash_version(
version.package.category, version.package.name, version.version,
version.revision, version.slot, version.overlay
)
self._cache['versions'][key] = version
def handle(self, *args, **options):
@ -119,7 +126,8 @@ class Command(BaseCommand):
cmd = ['eix', '-!']
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
output = output.strip().strip('\n').split('\n')
overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
@ -139,9 +147,10 @@ class Command(BaseCommand):
env = os.environ
env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages', '-x']
if query:
cmd.extend(['--exact', query])
cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages',
'-x']
if query:
cmd.extend(['--exact', query])
if self.options['all']:
if not self.options['quiet']:
@ -151,7 +160,8 @@ class Command(BaseCommand):
if not self.options['quiet']:
self.stdout.write('done\n')
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
output = output.strip().strip('\n')
if len(output) == 0:
@ -166,17 +176,23 @@ class Command(BaseCommand):
else:
Package.objects.filter(name=query).delete()
else:
sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % query))
sys.stderr.write(
self.style.ERROR(
"Unknown package '%s'\n" % query
)
)
return
output = output.split('\n')
output = output.split('\n')
packages = {}
line_re = re.compile(r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$')
line_re = re.compile(
r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$'
)
package = None
for line in output:
for line in output:
match = line_re.match(line)
if not match:
@ -190,7 +206,8 @@ class Command(BaseCommand):
packages['%s/%s' % (cat, pkg)] = True
if not package or not (cat == package.category and pkg == package.name):
if not package or not \
(cat == package.category and pkg == package.name):
package = self.store_package(cat, pkg)
self.store_version(package, cpv, slot, overlay)
@ -208,16 +225,23 @@ class Command(BaseCommand):
obj = self.cache_get_package(cat, pkg)
if not obj:
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
obj, created = Package.objects.get_or_create(
category=cat,
name=pkg
)
self.cache_store_package(obj)
if created:
if not self.options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
' Set all versions dead, then set found versions alive and delete old versions '
# Set all versions dead, then set found versions alive and
# delete old versions
if not self.options['all']:
Version.objects.filter(package=obj, packaged=True).update(alive=False)
Version.objects.filter(
package=obj,
packaged=True
).update(alive=False)
return obj
@ -232,11 +256,15 @@ class Command(BaseCommand):
overlay = 'gentoo'
created = False
obj = self.cache_get_version(package.category, package.name, ver, rev, slot, overlay)
obj = self.cache_get_version(
package.category, package.name, ver, rev, slot, overlay
)
if not obj:
obj, created = Version.objects.get_or_create(package=package, slot=slot,
revision=rev, version=ver,
overlay=overlay)
obj, created = Version.objects.get_or_create(
package=package, slot=slot,
revision=rev, version=ver,
overlay=overlay
)
obj.alive = True
obj.packaged = True
@ -245,7 +273,8 @@ class Command(BaseCommand):
if created:
self.cache_store_version(obj)
''' nothing to do (note: it can't be an upstream version because overlay can't be empty here) '''
# nothing to do (note: it can't be an upstream version because
# overlay can't be empty here)
if not created:
return
@ -287,8 +316,10 @@ class Command(BaseCommand):
if self.options['no-log']:
continue
entry = VersionLog.objects.create(package=version.package,
action=VersionLog.VERSION_REMOVED)
entry = VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED
)
entry.slot = version.slot
entry.revision = version.revision
entry.version = version.version
@ -296,4 +327,3 @@ class Command(BaseCommand):
entry.save()
Version.objects.filter(packaged=True, alive=False).delete()

38
euscanwww/djeuscan/management/commands/scan-upstream.py
View File

@ -1,19 +1,17 @@
import subprocess
import portage
import sys
import os
import re
from StringIO import StringIO
from datetime import datetime
from portage import versions
from optparse import make_option
from django.utils import timezone
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import Package, Version, EuscanResult, VersionLog
class Command(BaseCommand):
_overlays = {}
@ -60,7 +58,7 @@ class Command(BaseCommand):
elif args:
packages = list(args)
else:
packages = [ package[:-1] for package in sys.stdin.readlines() ]
packages = [package[:-1] for package in sys.stdin.readlines()]
self.scan(options, packages)
@ -83,8 +81,12 @@ class Command(BaseCommand):
def parse_output(self, options, output):
from portage.versions import _cp
package_re = re.compile(r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$')
version_re = re.compile(r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$')
package_re = re.compile(
r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$'
)
version_re = re.compile(
r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$'
)
package = None
log = ""
@ -124,7 +126,6 @@ class Command(BaseCommand):
obj.datetime = timezone.now()
obj.save()
def store_package(self, options, cpv):
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
@ -133,15 +134,16 @@ class Command(BaseCommand):
if created and not options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
' Set all versions dead, then set found versions alive and delete old versions '
# Set all versions dead, then set found versions alive and
# delete old versions
Version.objects.filter(package=obj, packaged=False).update(alive=False)
return obj
def store_version(self, options, package, ver, url):
obj, created = Version.objects.get_or_create(package=package, slot='',
revision='r0', version=ver,
overlay='')
obj, created = Version.objects.get_or_create(
package=package, slot='', revision='r0', version=ver, overlay=''
)
obj.alive = True
obj.urls = url
@ -155,7 +157,10 @@ class Command(BaseCommand):
if not options['quiet']:
sys.stdout.write('+ [u] %s %s\n' % (obj, url))
entry = VersionLog.objects.create(package=package, action=VersionLog.VERSION_ADDED)
entry = VersionLog.objects.create(
package=package,
action=VersionLog.VERSION_ADDED
)
entry.slot = ''
entry.revision = 'r0'
entry.version = ver
@ -165,12 +170,14 @@ class Command(BaseCommand):
package.n_versions += 1
package.save()
@commit_on_success
def purge_versions(self, options):
' For each dead versions '
for version in Version.objects.filter(packaged=False, alive=False):
entry = VersionLog.objects.create(package=version.package, action=VersionLog.VERSION_REMOVED)
entry = VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED
)
entry.slot = version.slot
entry.revision = version.revision
entry.version = version.version
@ -183,4 +190,3 @@ class Command(BaseCommand):
if not options['quiet']:
sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
Version.objects.filter(packaged=False, alive=False).delete()

55
euscanwww/djeuscan/management/commands/update-counters.py
View File

@ -1,10 +1,7 @@
import datetime
from optparse import make_option
from django.db.models import Count, Sum
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from django.utils import timezone
from djeuscan.models import Package, Herd, Maintainer, Version
@ -13,6 +10,7 @@ from djeuscan import charts
from distutils.version import StrictVersion, LooseVersion
def compare_versions(version1, version2):
try:
return cmp(StrictVersion(version1), StrictVersion(version2))
@ -20,6 +18,7 @@ def compare_versions(version1, version2):
except ValueError:
return cmp(LooseVersion(version1), LooseVersion(version2))
class Command(BaseCommand):
_overlays = {}
help = 'Update counters'
@ -98,7 +97,8 @@ class Command(BaseCommand):
return
if version['version'].startswith('9999'):
return
if compare_versions(storage[key]['version'], version['version']) < 0:
if compare_versions(storage[key]['version'],
version['version']) < 0:
storage[key] = version
if not options['fast']:
@ -125,25 +125,36 @@ class Command(BaseCommand):
package.n_packaged = n_packaged.get(package.id, 0)
package.n_overlay = n_overlay.get(package.id, 0)
default = {'id' : None}
package.last_version_gentoo_id = last_versions_gentoo.get(package.id, default)['id']
package.last_version_overlay_id = last_versions_overlay.get(package.id, default)['id']
package.last_version_upstream_id = last_versions_upstream.get(package.id, default)['id']
default = {'id': None}
package.last_version_gentoo_id = last_versions_gentoo.get(
package.id, default
)['id']
package.last_version_overlay_id = last_versions_overlay.get(
package.id, default
)['id']
package.last_version_upstream_id = last_versions_upstream.get(
package.id, default
)['id']
package.save()
n_packages_gentoo = int(package.n_packaged == package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged + package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay < package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged \
+ package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay \
< package.n_versions)
def update_row(storage, key):
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_outdated += n_packages_outdated
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
def update_log(storage, qs):
for row in qs:
update_row(storage, row['id'])
@ -153,13 +164,15 @@ class Command(BaseCommand):
update_log(maintainers, package.maintainers.all().values('id'))
update_row(categories, package.category)
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
if options['nolog']:
return

64
euscanwww/djeuscan/models.py
View File

@ -1,5 +1,6 @@
from django.db import models
class Herd(models.Model):
herd = models.CharField(max_length=128, unique=True)
email = models.CharField(max_length=128, blank=True, null=True)
@ -9,6 +10,7 @@ class Herd(models.Model):
return '%s <%s>' % (self.herd, self.email)
return self.herd
class Maintainer(models.Model):
name = models.CharField(max_length=128)
email = models.CharField(max_length=128, unique=True)
@ -16,6 +18,7 @@ class Maintainer(models.Model):
def __unicode__(self):
return '%s <%s>' % (self.name, self.email)
class Package(models.Model):
category = models.CharField(max_length=128)
name = models.CharField(max_length=128)
@ -30,15 +33,18 @@ class Package(models.Model):
n_overlay = models.IntegerField(default=0)
' And we also pre-compute last versions '
last_version_gentoo = models.ForeignKey('Version', blank=True, null=True,
related_name="last_version_gentoo",
on_delete=models.SET_NULL)
last_version_overlay = models.ForeignKey('Version', blank=True, null=True,
related_name="last_version_overlay",
on_delete=models.SET_NULL)
last_version_upstream = models.ForeignKey('Version', blank=True, null=True,
related_name="last_version_upstream",
on_delete=models.SET_NULL)
last_version_gentoo = models.ForeignKey(
'Version', blank=True, null=True, related_name="last_version_gentoo",
on_delete=models.SET_NULL
)
last_version_overlay = models.ForeignKey(
'Version', blank=True, null=True, related_name="last_version_overlay",
on_delete=models.SET_NULL
)
last_version_upstream = models.ForeignKey(
'Version', blank=True, null=True, related_name="last_version_upstream",
on_delete=models.SET_NULL
)
def __unicode__(self):
return '%s/%s' % (self.category, self.name)
@ -46,6 +52,7 @@ class Package(models.Model):
class Meta:
unique_together = ['category', 'name']
class Version(models.Model):
package = models.ForeignKey(Package)
slot = models.CharField(max_length=128)
@ -57,13 +64,15 @@ class Version(models.Model):
alive = models.BooleanField(default=True, db_index=True)
def __unicode__(self):
return '%s/%s-%s-%s:%s [%s]' % (self.package.category, self.package.name,
self.version, self.revision, self.slot,
self.overlay)
return '%s/%s-%s-%s:%s [%s]' % (
self.package.category, self.package.name, self.version,
self.revision, self.slot, self.overlay
)
class Meta:
unique_together = ['package', 'slot', 'revision', 'version', 'overlay']
class VersionLog(models.Model):
VERSION_ADDED = 1
VERSION_REMOVED = 2
@ -87,58 +96,67 @@ class VersionLog(models.Model):
def __unicode__(self):
txt = '+ ' if self.action == self.VERSION_ADDED else '- '
txt += '%s/%s-%s-%s:%s [%s]' % (self.package.category, self.package.name,
self.version, self.revision, self.slot,
self.overlay if self.overlay else '<upstream>')
txt += '%s/%s-%s-%s:%s [%s]' % (
self.package.category, self.package.name, self.version,
self.revision, self.slot,
self.overlay if self.overlay else '<upstream>'
)
return txt
class EuscanResult(models.Model):
package = models.ForeignKey(Package)
datetime = models.DateTimeField()
result = models.TextField(blank=True)
# Keep data for charts
class Log(models.Model):
datetime = models.DateTimeField()
' Packages up to date in the main portage tree '
n_packages_gentoo = models.IntegerField(default=0)
n_packages_gentoo = models.IntegerField(default=0)
' Packages up to date in an overlay '
n_packages_overlay = models.IntegerField(default=0)
n_packages_overlay = models.IntegerField(default=0)
' Packages outdated '
n_packages_outdated = models.IntegerField(default=0)
' Versions in the main portage tree '
n_versions_gentoo = models.IntegerField(default=0)
n_versions_gentoo = models.IntegerField(default=0)
' Versions in overlays '
n_versions_overlay = models.IntegerField(default=0)
n_versions_overlay = models.IntegerField(default=0)
' Upstream versions, not in the main tree or overlays '
n_versions_upstream = models.IntegerField(default=0)
def __unicode__(self):
return u'[%d:%d:%d] [%d:%d:%d]' % \
(self.n_packages_gentoo, self.n_packages_overlay, self.n_packages_outdated, \
self.n_versions_gentoo, self.n_versions_overlay, self.n_versions_upstream)
return u'[%d:%d:%d] [%d:%d:%d]' % (
self.n_packages_gentoo, self.n_packages_overlay,
self.n_packages_outdated, self.n_versions_gentoo,
self.n_versions_overlay, self.n_versions_upstream
)
class WorldLog(Log):
def __unicode__(self):
return u'world ' + Log.__unicode__(self)
class CategoryLog(Log):
category = models.CharField(max_length=128)
def __unicode__(self):
return u'%s %s' % (self.category, Log.__unicode__(self))
class HerdLog(Log):
herd = models.ForeignKey(Herd)
def __unicode__(self):
return u'%s %s' % (self.herd, Log.__unicode__(self))
class MaintainerLog(Log):
maintainer = models.ForeignKey(Maintainer)
def __unicode__(self):
return u'%s %s' % (self.maintainer, Log.__unicode__(self))

3
euscanwww/djeuscan/templatetags/div.py
View File

@ -2,7 +2,8 @@ from django import template
register = template.Library()
def div(value, arg=None):
return float(value)/float(arg)
return float(value) / float(arg)
register.filter('div', div)

14
euscanwww/djeuscan/templatetags/math_.py
View File

@ -1,11 +1,14 @@
from django.template import Node, Library
from django.template import Library
import math
register = Library()
# taken from http://lybniz2.sourceforge.net/safeeval.html
# make a list of safe functions
math_safe_list = ['acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'cosh', 'degrees', 'e', 'exp', 'fabs', 'floor', 'fmod', 'frexp', 'hypot', 'ldexp', 'log', 'log10', 'modf', 'pi', 'pow', 'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh']
math_safe_list = ['acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'cosh',
'degrees', 'e', 'exp', 'fabs', 'floor', 'fmod', 'frexp',
'hypot', 'ldexp', 'log', 'log10', 'modf', 'pi', 'pow',
'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh']
# use the list to filter the local namespace
math_safe_dict = dict([(k, getattr(math, k)) for k in math_safe_list])
@ -13,11 +16,13 @@ math_safe_dict = dict([(k, getattr(math, k)) for k in math_safe_list])
# add any needed builtins back in.
math_safe_dict['abs'] = abs
@register.filter('math')
def math_(lopr, expr):
"""Evals a math expression and returns it's value.
"$1" is a placeholder. Insert "$1" in the expression where the value is to be used. All math functions such as abs, sin, cos, floor are supported.
"$1" is a placeholder. Insert "$1" in the expression where the value is
to be used. All math functions such as abs, sin, cos, floor are supported.
Example,
a. You will be redirected in {{ seconds|math:"$1 / 60.0" }} minutes
b. Square of {{ x }} is {{ x|math:"$1 * $1" }}
@ -25,5 +30,6 @@ def math_(lopr, expr):
d. Given x = {{ x }}, (2 + x) * 6 = {{ x|math:"(2 + $1) * 6" }}
"""
if lopr:
return eval(expr.replace('$1', str(lopr)), {"__builtins__": None}, math_safe_dict)
return eval(expr.replace('$1', str(lopr)), {"__builtins__": None},
math_safe_dict)
return ''

3
euscanwww/djeuscan/templatetags/mul.py
View File

@ -2,7 +2,8 @@ from django import template
register = template.Library()
def mul(value, arg=None):
return value*arg
return value * arg
register.filter('mul', mul)

3
euscanwww/djeuscan/templatetags/packages.py
View File

@ -2,16 +2,19 @@ from django import template
register = template.Library()
@register.inclusion_tag('euscan/_packages.html', takes_context=True)
def packages(context, packages):
context['packages'] = packages
return context
@register.inclusion_tag('euscan/_package_cols.html', takes_context=True)
def package_cols(context, infos):
context['infos'] = infos
return context
@register.inclusion_tag('euscan/_package_bar.html', takes_context=True)
def package_bar(context, infos):
context['infos'] = infos

3
euscanwww/djeuscan/templatetags/sub.py
View File

@ -2,7 +2,8 @@ from django import template
register = template.Library()
def sub(value, arg=None):
return value-arg
return value - arg
register.filter('sub', sub)

7
euscanwww/djeuscan/templatetags/timedelta.py
View File

@ -6,6 +6,7 @@ from datetime import datetime
register = template.Library()
def timedelta(value, arg=None):
if not value:
return ''
@ -16,8 +17,8 @@ def timedelta(value, arg=None):
if settings.USE_TZ:
cmp = make_aware(cmp, get_default_timezone())
if value > cmp:
return "in %s" % timesince(cmp,value)
return "in %s" % timesince(cmp, value)
else:
return "%s ago" % timesince(value,cmp)
return "%s ago" % timesince(value, cmp)
register.filter('timedelta',timedelta)
register.filter('timedelta', timedelta)

2
euscanwww/djeuscan/tests.py
View File

@ -7,6 +7,7 @@ Replace these with more appropriate tests for your application.
from django.test import TestCase
class SimpleTest(TestCase):
def test_basic_addition(self):
"""
@ -20,4 +21,3 @@ Another way to test that 1 + 1 is equal to 2.
>>> 1 + 1 == 2
True
"""}

23
euscanwww/djeuscan/urls.py
View File

@ -1,29 +1,36 @@
from django.conf.urls.defaults import *
from feeds import *
from django.conf.urls.defaults import url, patterns, include
from feeds import PackageFeed, CategoryFeed, HerdFeed, MaintainerFeed, \
GlobalFeed
package_patterns = patterns('djeuscan.views',
url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/feed/$', PackageFeed(), name='package_feed'),
url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/feed/$',
PackageFeed(), name='package_feed'),
(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/$', 'package'),
)
categories_patterns = patterns('djeuscan.views',
(r'^(?P<category>[\w+][\w+.-]*)/(view/)?$', 'category'),
url(r'^(?P<category>[\w+][\w+.-]*)/feed/$', CategoryFeed(), name='category_feed'),
(r'^(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', 'chart_category'),
url(r'^(?P<category>[\w+][\w+.-]*)/feed/$', CategoryFeed(),
name='category_feed'),
(r'^(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$',
'chart_category'),
(r'^$', 'categories'),
)
herds_patterns = patterns('djeuscan.views',
(r'^(?P<herd>[\@\{\}\w+.-]*)/(view/)?$', 'herd'),
url(r'^(?P<herd>[\@\{\}\w+.-]*)/feed/$', HerdFeed(), name='herd_feed'),
(r'^(?P<herd>[\@\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', 'chart_herd'),
(r'^(?P<herd>[\@\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$',
'chart_herd'),
(r'^$', 'herds'),
)
maintainers_patterns = patterns('djeuscan.views',
(r'^(?P<maintainer_id>\d+)/(view/)?$', 'maintainer'),
url(r'^(?P<maintainer_id>\d+)/feed/$', MaintainerFeed(), name='maintainer_feed'),
(r'^(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$', 'chart_maintainer'),
url(r'^(?P<maintainer_id>\d+)/feed/$', MaintainerFeed(),
name='maintainer_feed'),
(r'^(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$',
'chart_maintainer'),
(r'^$', 'maintainers'),
)

117
euscanwww/djeuscan/views.py
View File

@ -1,5 +1,5 @@
from annoying.decorators import render_to
from django.http import HttpResponse, Http404
from django.http import Http404
from django.shortcuts import get_object_or_404
from django.db.models import Sum, Max
@ -10,84 +10,117 @@ import charts
""" Views """
@render_to('euscan/index.html')
def index(request):
ctx = {}
ctx['n_packaged'] = charts.xint(Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum'])
ctx['n_overlay'] = charts.xint(Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum'])
ctx['n_versions'] = charts.xint(Package.objects.aggregate(Sum('n_versions'))['n_versions__sum'])
ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - ctx['n_overlay']
ctx['n_packaged'] = charts.xint(
Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum']
)
ctx['n_overlay'] = charts.xint(
Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum']
)
ctx['n_versions'] = charts.xint(
Package.objects.aggregate(Sum('n_versions'))['n_versions__sum']
)
ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - \
ctx['n_overlay']
ctx['n_packages'] = Package.objects.count()
ctx['n_herds'] = Herd.objects.count()
ctx['n_maintainers'] = Maintainer.objects.count()
try:
ctx['last_scan'] = EuscanResult.objects.get(id=EuscanResult.objects.aggregate(Max('id'))['id__max']).datetime
ctx['last_scan'] = EuscanResult.objects.get(
id=EuscanResult.objects.aggregate(Max('id'))['id__max']
).datetime
except EuscanResult.DoesNotExist:
ctx['last_scan'] = None
return ctx
@render_to('euscan/logs.html')
def logs(request):
return {}
@render_to('euscan/categories.html')
def categories(request):
categories = Package.objects.values('category').annotate(n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
categories = Package.objects.values('category').annotate(
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')
)
return {'categories': categories}
return { 'categories' : categories }
@render_to('euscan/category.html')
def category(request, category):
packages = Package.objects.filter(category=category)
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
packages = packages.select_related(
'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
)
print dir(packages[0])
if not packages:
raise Http404
return { 'category' : category, 'packages' : packages }
return {'category': category, 'packages': packages}
@render_to('euscan/herds.html')
def herds(request):
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of 'INNER JOIN'
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of
# 'INNER JOIN'
herds = Package.objects.filter(herds__isnull=False)
herds = herds.values('herds__herd').annotate(n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
return { 'herds' : herds }
herds = herds.values('herds__herd').annotate(
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
return {'herds': herds}
@render_to('euscan/herd.html')
def herd(request, herd):
herd = get_object_or_404(Herd, herd=herd)
packages = Package.objects.filter(herds__id=herd.id)
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
return { 'herd' : herd, 'packages' : packages }
packages = packages.select_related(
'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
)
return {'herd': herd, 'packages': packages}
@render_to('euscan/maintainers.html')
def maintainers(request):
maintainers = Package.objects.filter(maintainers__isnull=False)
maintainers = maintainers.values('maintainers__id', 'maintainers__name', 'maintainers__email')
maintainers = maintainers.annotate(n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
maintainers = maintainers.values(
'maintainers__id', 'maintainers__name', 'maintainers__email'
)
maintainers = maintainers.annotate(
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')
)
return {'maintainers': maintainers}
return { 'maintainers' : maintainers }
@render_to('euscan/maintainer.html')
def maintainer(request, maintainer_id):
maintainer = get_object_or_404(Maintainer, id=maintainer_id)
packages = Package.objects.filter(maintainers__id=maintainer.id)
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
return { 'maintainer' : maintainer, 'packages' : packages }
packages = packages.select_related(
'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
)
return {'maintainer': maintainer, 'packages': packages}
@render_to('euscan/overlays.html')
def overlays(request):
overlays = Package.objects.values('version__overlay')
overlays = overlays.exclude(version__overlay='')
overlays = overlays.distinct()
return { 'overlays' : overlays }
return {'overlays': overlays}
@render_to('euscan/overlay.html')
def overlay(request, overlay):
@ -97,7 +130,8 @@ def overlay(request, overlay):
packages = packages.filter(version__overlay=overlay).distinct()
if not packages:
raise Http404
return { 'overlay' : overlay, 'packages' : packages }
return {'overlay': overlay, 'packages': packages}
@render_to('euscan/package.html')
def package(request, category, package):
@ -120,19 +154,23 @@ def package(request, category, package):
packaged = sorted(packaged, key=version_key)
upstream = sorted(upstream, key=version_key)
log = EuscanResult.objects.filter(package=package).order_by('-datetime')[:1]
log = EuscanResult.objects.filter(package=package).\
order_by('-datetime')[:1]
log = log[0] if log else None
vlog = VersionLog.objects.filter(package=package).order_by('-id')
return { 'package' : package, 'packaged' : packaged,
'upstream' : upstream, 'log' : log, 'vlog' : vlog }
return {'package': package, 'packaged': packaged,
'upstream': upstream, 'log': log, 'vlog': vlog}
@render_to('euscan/world.html')
def world(request):
world_form = WorldForm()
packages_form = PackagesForm()
return { 'world_form' : world_form,
'packages_form' : packages_form }
return {'world_form': world_form,
'packages_form': packages_form}
@render_to('euscan/world_scan.html')
def world_scan(request):
@ -157,28 +195,34 @@ def world_scan(request):
except:
pass
return { 'packages' : packages }
return {'packages': packages}
@render_to("euscan/about.html")
def about(request):
return {}
@render_to("euscan/api.html")
def api(request):
return {}
@render_to("euscan/statistics.html")
def statistics(request):
return {}
def chart(request, **kwargs):
from django.views.static import serve
chart = kwargs['chart'] if 'chart' in kwargs else None
if 'maintainer_id' in kwargs:
kwargs['maintainer'] = get_object_or_404(Maintainer, id=kwargs['maintainer_id'])
kwargs['maintainer'] = get_object_or_404(
Maintainer,
id=kwargs['maintainer_id']
)
if 'herd' in kwargs:
kwargs['herd'] = get_object_or_404(Herd, herd=kwargs['herd'])
@ -202,11 +246,14 @@ def chart(request, **kwargs):
return serve(request, path, document_root=charts.CHARTS_ROOT)
def chart_maintainer(request, **kwargs):
return chart(request, **kwargs)
def chart_herd(request, **kwargs):
return chart(request, **kwargs)
def chart_category(request, **kwargs):
return chart(request, **kwargs)