src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

euscanwww: charts, about, etc...

Browse Source 
Signed-off-by: Corentin Chary <corentincj@iksaif.net>
This commit is contained in:
Corentin Chary
2011-04-25 22:27:32 +02:00
parent 13dd433996
commit 0aba96f66f
30 changed files with 504 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
euscanwww/euscan/charts.py Normal file
View File

@ -0,0 +1,115 @@
import os.path
import time
from euscanwww import settings
from django.db.models import F, Sum, Max
from euscan.models import Version, Package, Herd, Maintainer
from euscan.models import CategoryLog
import pylab
import matplotlib
CHARTS_ROOT = os.path.join(settings.MEDIA_ROOT, "charts")
CHARTS_URL = os.path.join(settings.MEDIA_URL, "charts")
CHARTS_TTL = (24 * 60 * 60)
pylab.rcParams['font.size'] = 10.0
pylab.rcParams['axes.titlesize'] = 10.0
pylab.rcParams['xtick.labelsize'] = 8.0
pylab.rcParams['legend.fontsize'] = 8.0
def xint(i):
try:
return int(i)
except:
return 0
def chart_alive(name):
path = os.path.join(CHARTS_ROOT, name)
if not os.path.exists(path):
return False
if os.path.getmtime(__file__) > os.path.getmtime(path):
return False
if os.path.getmtime(path) + CHARTS_TTL < time.time():
return False
return True
def chart_name(name, **kwargs):
if 'category' in kwargs and kwargs['category']:
name += '-%s' % kwargs['category']
if 'herd' in kwargs and kwargs['herd']:
name += '-h-%d' % kwargs['herd'].id
if 'maintainer' in kwargs and kwargs['maintainer']:
name += '-m-%d' % kwargs['maintainer'].id
return name + ".png"
def packages(**kwargs):
packages = Package.objects
if 'category' in kwargs and kwargs['category']:
packages = packages.filter(category=kwargs['category'])
if 'herd' in kwargs and kwargs['herd']:
packages = packages.filter(herds__id=kwargs['herd'].id)
if 'maintainer' in kwargs and kwargs['maintainer']:
packages = packages.filter(maintainers__id=kwargs['maintainer'].id)
return packages
def cached_pylab_chart(f):
def new_f(*args, **kwds):
name = chart_name(f.func_name, **kwds)
if not chart_alive(name):
f(*args, **kwds)
pylab.savefig(os.path.join(CHARTS_ROOT, name))
pylab.close()
return name
new_f.func_name = f.func_name
return new_f
@cached_pylab_chart
def pie_versions(**kwargs):
n_packaged = xint(packages(**kwargs).aggregate(Sum('n_packaged'))['n_packaged__sum'])
n_overlay = xint(packages(**kwargs).aggregate(Sum('n_overlay'))['n_overlay__sum'])
n_versions = xint(packages(**kwargs).aggregate(Sum('n_versions'))['n_versions__sum'])
n_upstream = n_versions - n_packaged - n_overlay
pylab.figure(1, figsize=(3.5,3.5))
if n_overlay:
labels = 'Gentoo', 'Overlays', 'Upstream'
fracs = [n_packaged, n_overlay, n_upstream]
colors = '#008000', '#0B17FD', '#FF0000'
else:
labels = 'Gentoo', 'Upstream'
fracs = [n_packaged, n_upstream]
colors = '#008000', '#FF0000'
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', shadow=True)
pylab.title('Versions', bbox={'facecolor':'0.8', 'pad':5})
@cached_pylab_chart
def pie_packages(**kwargs):
n_packages = packages(**kwargs).count()
n_packages_uptodate_main = packages(**kwargs).filter(n_versions=F('n_packaged')).count()
n_packages_uptodate_all = packages(**kwargs).filter(n_versions=F('n_packaged') + F('n_overlay')).count()
n_packages_outdated = n_packages - n_packages_uptodate_all
n_packages_uptodate_ovl = n_packages_uptodate_all - n_packages_uptodate_main
pylab.figure(1, figsize=(3.5,3.5))
if n_packages_uptodate_ovl:
labels = 'Ok (gentoo)', 'Ok (overlays)', 'Outdated'
fracs = [n_packages_uptodate_main, n_packages_uptodate_ovl, n_packages_outdated]
colors = '#008000', '#0B17FD', '#FF0000'
else:
labels = 'Ok (gentoo)', 'Outdated'
fracs = [n_packages_uptodate_main, n_packages_outdated]
colors = '#008000', '#FF0000'
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', shadow=True)
pylab.title('Packages', bbox={'facecolor':'0.8', 'pad':5})

2
euscanwww/euscan/management/commands/scan-metadata.py
View File

@ -62,7 +62,7 @@ class Command(BaseCommand):
matches = sorted(matches)
pkg = matches.pop()
if pkg.version == '9999' and len(matches):
if '9999' in pkg.version and len(matches):
pkg = matches.pop()
obj, created = Package.objects.get_or_create(category=pkg.category, name=pkg.name)

16
euscanwww/euscan/management/commands/scan-portage.py
View File

@ -103,14 +103,14 @@ class Command(BaseCommand):
slot = match.group('slot')
overlay = match.group('overlay')
if not package or not cpv.startswith(str(package)):
package = self.store_package(options, cpv)
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
if not package or not (cat == package.category and pkg == package.name):
package = self.store_package(options, cat, pkg)
self.store_version(options, package, cpv, slot, overlay)
def store_package(self, options, cpv):
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
def store_package(self, options, cat, pkg):
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
if created:
@ -121,6 +121,7 @@ class Command(BaseCommand):
Version.objects.filter(package=obj, packaged=True).delete()
obj.n_packaged = 0
obj.n_overlay = 0
obj.n_versions = Version.objects.filter(package=obj).count()
obj.save()
@ -144,7 +145,10 @@ class Command(BaseCommand):
overlay=overlay)
if created or not package.n_packaged:
package.n_packaged += 1
if overlay == 'gentoo':
package.n_packaged += 1
else:
package.n_overlay += 1
if created:
package.n_versions += 1

18
euscanwww/euscan/management/commands/update-counters.py
View File

@ -51,40 +51,44 @@ class Command(BaseCommand):
for package in Package.objects.all():
# Should not be needed, but can't hurt
package.n_versions = Version.objects.filter(package=package).count()
package.n_packaged != Version.objects.filter(package=package,packaged=True).count()
package.n_packaged = Version.objects.filter(package=package, packaged=True, overlay='gentoo').count()
package.n_overlay = Version.objects.filter(package=package, packaged=True).exclude(overlay='gentoo').count()
package.save()
for herd in package.herds.all():
herds[herd].n_packages += 1
herds[herd].n_versions += package.n_versions
herds[herd].n_packaged += package.n_packaged
herds[herd].n_overlay += package.n_overlay
for maintainer in package.maintainers.all():
maintainers[maintainer].n_packages += 1
maintainers[maintainer].n_versions += package.n_versions
maintainers[maintainer].n_packaged += package.n_packaged
maintainers[maintainer].n_overlay += package.n_overlay
categories[package.category].n_packages += 1
categories[package.category].n_versions += package.n_versions
categories[package.category].n_packaged += package.n_packaged
categories[package.category].n_overlay += package.n_overlay
for clog in categories.values():
if not options['quiet']:
self.stdout.write('[c] %s - [%d, %d/%d]\n' %
self.stdout.write('[c] %s - [%d, %d/%d/%d]\n' %
(clog.category, clog.n_packages,
clog.n_packaged, clog.n_versions))
clog.n_packaged, clog.n_overlay, clog.n_versions))
clog.save()
for hlog in herds.values():
if not options['quiet']:
self.stdout.write('[h] %s - [%d, %d/%d]\n' %
self.stdout.write('[h] %s - [%d, %d/%d/%d]\n' %
(hlog.herd, hlog.n_packages,
hlog.n_packaged, hlog.n_versions))
hlog.n_packaged, hlog.n_overlay, hlog.n_versions))
hlog.save()
for mlog in maintainers.values():
if not options['quiet']:
self.stdout.write('[m] %s - [%d, %d/%d]\n' %
self.stdout.write('[m] %s - [%d, %d/%d/%d]\n' %
(mlog.maintainer, mlog.n_packages,
mlog.n_packaged, mlog.n_versions))
mlog.n_packaged, mlog.n_overlay, mlog.n_versions))
mlog.save()

115
euscanwww/euscan/migrations/0003_auto__add_field_categorylog_n_overlay__add_field_package_n_overlay__ad.py Normal file
View File

@ -0,0 +1,115 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'CategoryLog.n_overlay'
db.add_column('euscan_categorylog', 'n_overlay', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
# Adding field 'Package.n_overlay'
db.add_column('euscan_package', 'n_overlay', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
# Adding field 'HerdLog.n_overlay'
db.add_column('euscan_herdlog', 'n_overlay', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
# Adding field 'MaintainerLog.n_overlay'
db.add_column('euscan_maintainerlog', 'n_overlay', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
def backwards(self, orm):
# Deleting field 'CategoryLog.n_overlay'
db.delete_column('euscan_categorylog', 'n_overlay')
# Deleting field 'Package.n_overlay'
db.delete_column('euscan_package', 'n_overlay')
# Deleting field 'HerdLog.n_overlay'
db.delete_column('euscan_herdlog', 'n_overlay')
# Deleting field 'MaintainerLog.n_overlay'
db.delete_column('euscan_maintainerlog', 'n_overlay')
models = {
'euscan.categorylog': {
'Meta': {'object_name': 'CategoryLog'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'euscan.euscanresult': {
'Meta': {'object_name': 'EuscanResult'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'euscan.herd': {
'Meta': {'object_name': 'Herd'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'euscan.herdlog': {
'Meta': {'object_name': 'HerdLog'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Herd']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'euscan.maintainer': {
'Meta': {'object_name': 'Maintainer'},
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'euscan.maintainerlog': {
'Meta': {'object_name': 'MaintainerLog'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Maintainer']"}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'euscan.package': {
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
'homepage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'euscan.version': {
'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
}
}
complete_apps = ['euscan']

6
euscanwww/euscan/models.py
View File

@ -27,6 +27,7 @@ class Package(models.Model):
# For performance, we keep pre-computed counters
n_versions = models.IntegerField(default=0)
n_packaged = models.IntegerField(default=0)
n_overlay = models.IntegerField(default=0)
def __unicode__(self):
return '%s/%s' % (self.category, self.name)
@ -40,7 +41,7 @@ class Version(models.Model):
revision = models.CharField(max_length=128)
version = models.CharField(max_length=128)
packaged = models.BooleanField()
overlay = models.CharField(max_length=128)
overlay = models.CharField(max_length=128, default='gentoo')
urls = models.TextField(blank=True)
def __unicode__(self):
@ -64,6 +65,7 @@ class CategoryLog(models.Model):
n_packages = models.IntegerField(default=0)
n_versions = models.IntegerField(default=0)
n_packaged = models.IntegerField(default=0)
n_overlay = models.IntegerField(default=0)
class HerdLog(models.Model):
herd = models.ForeignKey(Herd)
@ -72,6 +74,7 @@ class HerdLog(models.Model):
n_packages = models.IntegerField(default=0)
n_versions = models.IntegerField(default=0)
n_packaged = models.IntegerField(default=0)
n_overlay = models.IntegerField(default=0)
class MaintainerLog(models.Model):
maintainer = models.ForeignKey(Maintainer)
@ -80,3 +83,4 @@ class MaintainerLog(models.Model):
n_packages = models.IntegerField(default=0)
n_versions = models.IntegerField(default=0)
n_packaged = models.IntegerField(default=0)
n_overlay = models.IntegerField(default=0)

8
euscanwww/euscan/templatetags/div.py Normal file
View File

@ -0,0 +1,8 @@
from django import template
register = template.Library()
def div(value, arg=None):
return value/arg
register.filter('div', div)

8
euscanwww/euscan/templatetags/packages.py
View File

@ -5,3 +5,11 @@ register = template.Library()
@register.inclusion_tag('euscan/_packages.html')
def packages(packages):
return { 'packages' : packages }
@register.inclusion_tag('euscan/_package_cols.html')
def package_cols(infos):
return { 'infos' : infos }
@register.inclusion_tag('euscan/_package_bar.html')
def package_bar(infos):
return { 'infos' : infos }

7
euscanwww/euscan/urls.py
View File

@ -2,14 +2,19 @@ from django.conf.urls.defaults import *
urlpatterns = patterns('euscan.views',
(r'^$', 'index'),
(r'^logs/$', 'logs'),
(r'^about/$', 'about'),
(r'^statistics/$', 'statistics'),
(r'^statistics/charts/(?P<chart>[\w\-]+).png$', 'chart'),
(r'^world/$', 'world'),
(r'^world/scan/$', 'world_scan'),
(r'^categories/$', 'categories'),
(r'^categories/(?P<category>[\w+][\w+.-]*)/view/$', 'category'),
(r'^categories/(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', 'chart_category'),
(r'^herds/$', 'herds'),
(r'^herds/(?P<herd>[\{\}\w+.-]*)/view/$', 'herd'),
(r'^herds/(?P<herd>[\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', 'chart_herd'),
(r'^maintainers/$', 'maintainers'),
(r'^maintainers/(?P<maintainer_id>\d+)/view/$', 'maintainer'),
(r'^maintainers/(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$', 'chart_maintainer'),
(r'^package/(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/$', 'package'),
)

67
euscanwww/euscan/views.py
View File

@ -1,4 +1,5 @@
from annoying.decorators import render_to
from django.http import HttpResponse
from django.http import Http404
from django.shortcuts import get_object_or_404
from django.db.models import Sum, Max
@ -6,13 +7,17 @@ from django.db.models import Sum, Max
from euscan.models import Version, Package, Herd, Maintainer, EuscanResult
from euscan.forms import WorldForm, PackagesForm
import charts
""" Views """
@render_to('euscan/index.html')
def index(request):
ctx = {}
ctx['n_packaged'] = Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum']
ctx['n_versions'] = Package.objects.aggregate(Sum('n_versions'))['n_versions__sum']
if ctx['n_versions'] is not None and ctx['n_packaged'] is not None:
ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged']
ctx['n_packaged'] = charts.xint(Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum'])
ctx['n_overlay'] = charts.xint(Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum'])
ctx['n_versions'] = charts.xint(Package.objects.aggregate(Sum('n_versions'))['n_versions__sum'])
ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - ctx['n_overlay']
ctx['n_packages'] = Package.objects.count()
ctx['n_herds'] = Herd.objects.count()
ctx['n_maintainers'] = Maintainer.objects.count()
@ -25,7 +30,10 @@ def logs(request):
@render_to('euscan/categories.html')
def categories(request):
categories = Package.objects.values('category').annotate(n_packaged=Sum('n_packaged'), n_versions=Sum('n_versions'))
categories = Package.objects.values('category').annotate(n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
return { 'categories' : categories }
@render_to('euscan/category.html')
@ -38,7 +46,10 @@ def category(request, category):
@render_to('euscan/herds.html')
def herds(request):
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of 'INNER JOIN'
herds = Package.objects.filter(herds__isnull=False).values('herds__herd').annotate(n_packaged=Sum('n_packaged'), n_versions=Sum('n_versions'))
herds = Package.objects.filter(herds__isnull=False)
herds = herds.values('herds__herd').annotate(n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
return { 'herds' : herds }
@render_to('euscan/herd.html')
@ -49,7 +60,12 @@ def herd(request, herd):
@render_to('euscan/maintainers.html')
def maintainers(request):
maintainers = Package.objects.filter(maintainers__isnull=False).values('maintainers__id', 'maintainers__name').annotate(n_packaged=Sum('n_packaged'), n_versions=Sum('n_versions'))
maintainers = Package.objects.filter(maintainers__isnull=False)
maintainers = maintainers.values('maintainers__id', 'maintainers__name')
maintainers = maintainers.annotate(n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
return { 'maintainers' : maintainers }
@render_to('euscan/maintainer.html')
@ -61,6 +77,7 @@ def maintainer(request, maintainer_id):
@render_to('euscan/package.html')
def package(request, category, package):
package = get_object_or_404(Package, category=category, name=package)
package.homepages = package.homepage.split(' ')
packaged = Version.objects.filter(package=package, packaged=True)
upstream = Version.objects.filter(package=package, packaged=False)
log = EuscanResult.objects.filter(package=package).order_by('-datetime')[:1]
@ -99,8 +116,42 @@ def world_scan(request):
packages.extend(Package.objects.filter(name=pkg))
except:
pass
print packages
return { 'packages' : packages }
@render_to("euscan/about.html")
def about(request):
return {}
@render_to("euscan/statistics.html")
def statistics(request):
return {}
def chart(request, **kwargs):
from django.views.static import serve
chart = kwargs['chart'] if 'chart' in kwargs else None
if 'maintainer_id' in kwargs:
kwargs['maintainer'] = get_object_or_404(Maintainer, id=kwargs['maintainer_id'])
if 'herd' in kwargs:
kwargs['herd'] = get_object_or_404(Herd, herd=kwargs['herd'])
if chart == 'pie-packages':
path = charts.pie_packages(**kwargs)
elif chart == 'pie-versions':
path = charts.pie_versions(**kwargs)
else:
raise Http404()
return serve(request, path, document_root=charts.CHARTS_ROOT)
def chart_maintainer(request, **kwargs):
return chart(request, **kwargs)
def chart_herd(request, **kwargs):
return chart(request, **kwargs)
def chart_category(request, **kwargs):
return chart(request, **kwargs)