src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

euscanwww: add freshness and last versions

Browse Source 
Signed-off-by: Corentin Chary <corentincj@iksaif.net>
This commit is contained in:
Corentin Chary 2012-03-02 18:01:46 +01:00
parent d899ffb849
commit ff33042f36
11 changed files with 277 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
euscanwww/api/handlers.py
View File

@ -108,7 +108,10 @@ class CategoriesHandler(AnonymousBaseHandler):
# /api/1.0/packages/by-herd/
class PackagesHandler(AnonymousBaseHandler):
allowed_methods = ('GET',)
fields = ('category', 'name', 'n_packaged', 'n_overlay', 'n_versions')
fields = ('category', 'name', 'n_packaged', 'n_overlay', 'n_versions',
('last_version_gentoo', ('version',)),
('last_version_overlay', ('version',)),
('last_version_upstream', ('version',)))
model = Package
@catch_and_return(ObjectDoesNotExist, rc.NOT_FOUND)
@ -117,15 +120,18 @@ class PackagesHandler(AnonymousBaseHandler):
if 'category' in kwargs:
packages = Package.objects.filter(category=kwargs['category'])
data = { 'category' : kwargs['category'], 'packages' : packages }
data = { 'category' : kwargs['category'] }
elif 'herd' in kwargs:
herd = Herd.objects.get(herd=kwargs['herd'])
packages = Package.objects.filter(herds__id=herd.id)
data = { 'herd' : herd, 'packages' : packages }
data = { 'herd' : herd }
elif 'maintainer_id' in kwargs:
maintainer = Maintainer.objects.get(id=kwargs['maintainer_id'])
packages = Package.objects.filter(maintainers__id=maintainer.id)
data = { 'maintainer' : maintainer, 'packages' : packages }
data = { 'maintainer' : maintainer }
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
data['packages'] = packages
if not data:
return rc.NOT_FOUND

167
euscanwww/euscan/management/commands/update-counters.py
View File

@ -5,6 +5,7 @@ from optparse import make_option
from django.db.models import Count, Sum
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from euscanwww.euscan.models import Package, Herd, Maintainer, Version
from euscanwww.euscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from euscanwww.euscan import charts
@ -19,6 +20,17 @@ class Command(BaseCommand):
dest='quiet',
default=False,
help='Be quiet'),
make_option('--fast',
action='store_true',
dest='fast',
default=False,
help='Skip sanity checks'),
make_option('--nolog',
action='store_true',
dest='nolog',
default=False,
help='Skip logs'),
)
@commit_on_success
@ -29,76 +41,118 @@ class Command(BaseCommand):
herds = {}
maintainers = {}
'''
Could be done using raw SQL queries, but I don't have time for that
right now ...
'''
wlog = None
wlog = WorldLog()
wlog.datetime = now
if not options['nolog']:
wlog = WorldLog()
wlog.datetime = now
for cat in Package.objects.values('category').distinct():
clog = CategoryLog()
clog.datetime = now
clog.category = cat['category']
categories[clog.category] = clog
for cat in Package.objects.values('category').distinct():
clog = CategoryLog()
clog.datetime = now
clog.category = cat['category']
categories[clog.category] = clog
for herd in Herd.objects.all():
hlog = HerdLog()
hlog.datetime = now
hlog.herd = herd
herds[herd] = hlog
for herd in Herd.objects.all():
hlog = HerdLog()
hlog.datetime = now
hlog.herd = herd
herds[herd.id] = hlog
for maintainer in Maintainer.objects.all():
mlog = MaintainerLog()
mlog.datetime = now
mlog.maintainer = maintainer
maintainers[maintainer] = mlog
for maintainer in Maintainer.objects.all():
mlog = MaintainerLog()
mlog.datetime = now
mlog.maintainer = maintainer
maintainers[maintainer.id] = mlog
for package in Package.objects.all():
# Should not be needed, but can't hurt
package.n_versions = Version.objects.filter(package=package).count()
package.n_packaged = Version.objects.filter(package=package, packaged=True, overlay='gentoo').count()
package.n_overlay = Version.objects.filter(package=package, packaged=True).exclude(overlay='gentoo').count()
package.save()
package_queryset = Package.objects.all()
n_versions = {}
n_packaged = {}
n_overlay = {}
last_versions_gentoo = {}
last_versions_overlay = {}
last_versions_upstream = {}
def add_safe(storage, key):
if key not in storage:
storage[key] = 1
else:
storage[key] += 1
def add_last_ver(storage, version):
key = version['package_id']
if key not in storage:
storage[key] = version
return
if version['version'].startswith('9999'):
return
if storage[key]['version'] < version['version']:
storage[key] = version
if not options['fast']:
attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
for version in Version.objects.all().values(*attrs):
overlay, packaged = version['overlay'], version['packaged']
package_id = version['package_id']
add_safe(n_versions, package_id)
if not packaged:
add_last_ver(last_versions_upstream, version)
continue
if overlay == 'gentoo':
add_safe(n_packaged, package_id)
add_last_ver(last_versions_gentoo, version)
else:
add_safe(n_overlay, package_id)
add_last_ver(last_versions_overlay, version)
for package in package_queryset.select_related('herds', 'maintainers'):
if not options['fast']:
package.n_versions = n_versions.get(package.id, 0)
package.n_packaged = n_packaged.get(package.id, 0)
package.n_overlay = n_overlay.get(package.id, 0)
default = {'id' : -1}
package.last_version_gentoo_id = last_versions_gentoo.get(package.id, default)['id']
package.last_version_overlay_id = last_versions_overlay.get(package.id, default)['id']
package.last_version_upstream_id = last_versions_upstream.get(package.id, default)['id']
package.save()
n_packages_gentoo = int(package.n_packaged == package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged + package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay < package.n_versions)
for herd in package.herds.all():
herds[herd].n_packages_gentoo += n_packages_gentoo
herds[herd].n_packages_overlay += n_packages_overlay
herds[herd].n_packages_outdated += n_packages_outdated
def update_row(storage, key):
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_outdated += n_packages_outdated
herds[herd].n_versions_gentoo += package.n_packaged
herds[herd].n_versions_overlay += package.n_overlay
herds[herd].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
def update_log(storage, qs):
for row in qs:
update_row(storage, row['id'])
for maintainer in package.maintainers.all():
maintainers[maintainer].n_packages_gentoo += n_packages_gentoo
maintainers[maintainer].n_packages_overlay += n_packages_overlay
maintainers[maintainer].n_packages_outdated += n_packages_outdated
if not options['nolog']:
update_log(herds, package.herds.all().values('id'))
update_log(maintainers, package.maintainers.all().values('id'))
update_row(categories, package.category)
maintainers[maintainer].n_versions_gentoo += package.n_packaged
maintainers[maintainer].n_versions_overlay += package.n_overlay
maintainers[maintainer].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated
categories[package.category].n_packages_gentoo += n_packages_gentoo
categories[package.category].n_packages_overlay += n_packages_overlay
categories[package.category].n_packages_outdated += n_packages_outdated
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
categories[package.category].n_versions_gentoo += package.n_packaged
categories[package.category].n_versions_overlay += package.n_overlay
categories[package.category].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
if options['nolog']:
return
for clog in categories.values():
if not options['quiet']:
@ -118,6 +172,5 @@ class Command(BaseCommand):
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save()
wlog.save()
charts.rrd_update('world', now, wlog)
wlog.save()

125
euscanwww/euscan/migrations/0009_auto__add_field_package_last_version_gentoo__add_field_package_last_ve.py Normal file
View File

@ -0,0 +1,125 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'Package.last_version_gentoo'
db.add_column('euscan_package', 'last_version_gentoo', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='last_version_gentoo', null=True, to=orm['euscan.Version']), keep_default=False)
# Adding field 'Package.last_version_overlay'
db.add_column('euscan_package', 'last_version_overlay', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='last_version_overlay', null=True, to=orm['euscan.Version']), keep_default=False)
# Adding field 'Package.last_version_upstream'
db.add_column('euscan_package', 'last_version_upstream', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='last_version_upstream', null=True, to=orm['euscan.Version']), keep_default=False)
def backwards(self, orm):
# Deleting field 'Package.last_version_gentoo'
db.delete_column('euscan_package', 'last_version_gentoo_id')
# Deleting field 'Package.last_version_overlay'
db.delete_column('euscan_package', 'last_version_overlay_id')
# Deleting field 'Package.last_version_upstream'
db.delete_column('euscan_package', 'last_version_upstream_id')
models = {
'euscan.categorylog': {
'Meta': {'object_name': 'CategoryLog', '_ormbases': ['euscan.Log']},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'euscan.euscanresult': {
'Meta': {'object_name': 'EuscanResult'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'euscan.herd': {
'Meta': {'object_name': 'Herd'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'euscan.herdlog': {
'Meta': {'object_name': 'HerdLog', '_ormbases': ['euscan.Log']},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Herd']"}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'euscan.log': {
'Meta': {'object_name': 'Log'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'euscan.maintainer': {
'Meta': {'object_name': 'Maintainer'},
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'euscan.maintainerlog': {
'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['euscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Maintainer']"})
},
'euscan.package': {
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
'homepage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'to': "orm['euscan.Version']"}),
'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'to': "orm['euscan.Version']"}),
'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'to': "orm['euscan.Version']"}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'euscan.version': {
'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'euscan.versionlog': {
'Meta': {'object_name': 'VersionLog'},
'action': ('django.db.models.fields.IntegerField', [], {}),
'datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 3, 2, 16, 32, 3, 656974)'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'euscan.worldlog': {
'Meta': {'object_name': 'WorldLog', '_ormbases': ['euscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'})
}
}
complete_apps = ['euscan']

8
euscanwww/euscan/models.py
View File

@ -30,6 +30,14 @@ class Package(models.Model):
n_packaged = models.IntegerField(default=0)
n_overlay = models.IntegerField(default=0)
' And we also pre-compute last versions '
last_version_gentoo = models.ForeignKey('Version', blank=True, null=True,
related_name="last_version_gentoo")
last_version_overlay = models.ForeignKey('Version', blank=True, null=True,
related_name="last_version_overlay")
last_version_upstream = models.ForeignKey('Version', blank=True, null=True,
related_name="last_version_upstream")
def __unicode__(self):
return '%s/%s' % (self.category, self.name)

2
euscanwww/euscan/templatetags/div.py
View File

@ -3,6 +3,6 @@ from django import template
register = template.Library()
def div(value, arg=None):
return value/arg
return float(value)/float(arg)
register.filter('div', div)

7
euscanwww/euscan/views.py
View File

@ -37,7 +37,8 @@ def categories(request):
@render_to('euscan/category.html')
def category(request, category):
packages = Package.objects.filter(category=category)
packages = Package.objects.filter(category=category).select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
print dir(packages[0])
if not packages:
raise Http404
return { 'category' : category, 'packages' : packages }
@ -94,8 +95,8 @@ def overlay(request, overlay):
def package(request, category, package):
package = get_object_or_404(Package, category=category, name=package)
package.homepages = package.homepage.split(' ')
packaged = Version.objects.filter(package=package, packaged=True)
upstream = Version.objects.filter(package=package, packaged=False)
packaged = Version.objects.filter(package=package, packaged=True).order_by('version', 'revision')
upstream = Version.objects.filter(package=package, packaged=False).order_by('version', 'revision')
log = EuscanResult.objects.filter(package=package).order_by('-datetime')[:1]
log = log[0] if log else None
vlog = VersionLog.objects.filter(package=package).order_by('-id')

6
euscanwww/templates/euscan/_package_cols.html
View File

@ -4,6 +4,7 @@
{% load div %}
<td>{{ infos.n_packaged }}</td>
{% if infos.n_overlay == 0 or infos.n_overlay <= infos.n_packaged %}
<td>
{% else %}{% if infos.n_overlay < infos.n_packaged %}
@ -13,6 +14,7 @@
{% endif %}{% endif %}
{{ infos.n_overlay }}
</td>
{% if infos.n_versions == infos.n_packaged|add:infos.n_overlay %}
<td>
{% else %}{% if infos.n_versions < infos.n_packaged|add:infos.n_overlay|mul:2 %}
@ -22,3 +24,7 @@
{% endif %}{% endif %}
{{ infos.n_versions|sub:infos.n_packaged|sub:infos.n_overlay }}
</td>
<td>
{{ infos.n_packaged|add:infos.n_overlay|div:infos.n_versions|mul:100|floatformat:"0" }}
</td>

10
euscanwww/templates/euscan/_packages.html
View File

@ -1,11 +1,18 @@
{% load packages %}
{% load sub %}
{% load div %}
{% load mul %}
<table id="table" class="display">
<thead>
<th>Package</th>
<th>Gentoo</th>
<th>Overlay</th>
<th>Upstream</th>
<th>Gentoo</th>
<th>Overlays</th>
<th>Unpackaged</th>
<th>Freshness</th>
</thead>
<tbody>
{% for package in packages %}
@ -16,6 +23,9 @@
</a>
{% package_bar package %}
</td>
<td>{{ package.last_version_gentoo.version }}</td>
<td>{{ package.last_version_overlay.version }}</td>
<td>{{ package.last_version_upstream.version }}</td>
{% package_cols package %}
</tr>
{% endfor %}

1
euscanwww/templates/euscan/categories.html
View File

@ -15,6 +15,7 @@
<th>Gentoo</th>
<th>Overlays</th>
<th>Unpackaged</th>
<th>Freshness</th>
{% if request.GET.extras %}
<th>Graphs</th>
{% endif %}

1
euscanwww/templates/euscan/herds.html
View File

@ -14,6 +14,7 @@
<th>Gentoo</th>
<th>Overlays</th>
<th>Unpackaged</th>
<th>Freshness</th>
{% if request.GET.extras %}
<th>Graphs</th>
{% endif %}

1
euscanwww/templates/euscan/maintainers.html
View File

@ -14,6 +14,7 @@
<th>Gentoo</th>
<th>Overlays</th>
<th>Unpackaged</th>
<th>Freshness</th>
{% if request.GET.extras %}
<th>Graphs</th>
{% endif %}