src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

euscanwww: finish charts

Browse Source 
Signed-off-by: Corentin Chary <corentincj@iksaif.net>
This commit is contained in:
Corentin Chary 2011-05-05 18:10:47 +02:00
parent 54f7771b43
commit 917a290e3a
10 changed files with 315 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,2 +1,4 @@
*~
*.pyc
euscanwww/rrd/*.rrd
euscanwww/media/charts/*.png

2
euscanwww/euscan/charts.py
View File

@ -52,6 +52,8 @@ def rrd_name(**kwargs):
return name
def chart_name(name, **kwargs):
name = name.replace('_', '-')
if 'category' in kwargs and kwargs['category']:
name += '-%s' % kwargs['category']
if 'herd' in kwargs and kwargs['herd']:

5
euscanwww/euscan/management/commands/regen-rrds.py
View File

@ -3,7 +3,7 @@ import datetime
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from euscanwww.euscan.models import Package, HerdLog, MaintainerLog, CategoryLog, Herd, Maintainer, Version
from euscanwww.euscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from euscanwww.euscan import charts
class Command(BaseCommand):
@ -11,6 +11,9 @@ class Command(BaseCommand):
help = 'Regenerate rrd database'
def handle(self, *args, **options):
for wlog in WorldLog.objects.all():
charts.rrd_update('world', wlog.datetime, wlog)
for clog in CategoryLog.objects.all():
charts.rrd_update('category-%s' % clog.category, clog.datetime, clog)

2
euscanwww/euscan/management/commands/scan-portage.py
View File

@ -122,7 +122,7 @@ class Command(BaseCommand):
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
packages['%s/%s' % (cat, pkg)] = True
continue
if not package or not (cat == package.category and pkg == package.name):
package = self.store_package(options, cat, pkg)

31
euscanwww/euscan/management/commands/update-counters.py
View File

@ -5,17 +5,10 @@ from optparse import make_option
from django.db.models import Count, Sum
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from euscanwww.euscan.models import Package, HerdLog, MaintainerLog, CategoryLog, Herd, Maintainer, Version
from euscanwww.euscan.models import Package, Herd, Maintainer, Version
from euscanwww.euscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from euscanwww.euscan import charts
class World:
n_packages_gentoo = 0
n_packages_overlay = 0
n_packages_outdated = 0
n_versions_gentoo = 0
n_versions_overlay = 0
n_versions_upstream = 0
class Command(BaseCommand):
_overlays = {}
help = 'Update counters'
@ -35,11 +28,13 @@ class Command(BaseCommand):
categories = {}
herds = {}
maintainers = {}
world = World()
# Could be done using raw SQL queries, but I don't have time for that
# right now ...
wlog = WorldLog()
wlog.datetime = now
for cat in Package.objects.values('category').distinct():
clog = CategoryLog()
clog.datetime = now
@ -95,13 +90,13 @@ class Command(BaseCommand):
categories[package.category].n_versions_overlay += package.n_overlay
categories[package.category].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
world.n_packages_gentoo += n_packages_gentoo
world.n_packages_overlay += n_packages_overlay
world.n_packages_outdated += n_packages_outdated
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated
world.n_versions_gentoo += package.n_packaged
world.n_versions_overlay += package.n_overlay
world.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
for clog in categories.values():
if not options['quiet']:
@ -121,4 +116,6 @@ class Command(BaseCommand):
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save()
charts.rrd_update('world', now, world)
wlog.save()
charts.rrd_update('world', now, wlog)

139
euscanwww/euscan/migrations/0005_auto__del_categorylog__del_herdlog__del_maintainerlog__add_log.py Normal file
View File

@ -0,0 +1,139 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Deleting model 'CategoryLog'
db.delete_table('euscan_categorylog')
# Deleting model 'HerdLog'
db.delete_table('euscan_herdlog')
# Deleting model 'MaintainerLog'
db.delete_table('euscan_maintainerlog')
# Adding model 'Log'
db.create_table('euscan_log', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('datetime', self.gf('django.db.models.fields.DateTimeField')()),
('n_packages_gentoo', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_packages_overlay', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_packages_outdated', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_versions_gentoo', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_versions_overlay', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_versions_upstream', self.gf('django.db.models.fields.IntegerField')(default=0)),
))
db.send_create_signal('euscan', ['Log'])
def backwards(self, orm):
# Adding model 'CategoryLog'
db.create_table('euscan_categorylog', (
('category', self.gf('django.db.models.fields.CharField')(max_length=128)),
('n_packages_gentoo', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_packages_outdated', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_packages_overlay', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_versions_upstream', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_versions_overlay', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_versions_gentoo', self.gf('django.db.models.fields.IntegerField')(default=0)),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('datetime', self.gf('django.db.models.fields.DateTimeField')()),
))
db.send_create_signal('euscan', ['CategoryLog'])
# Adding model 'HerdLog'
db.create_table('euscan_herdlog', (
('n_packages_gentoo', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_versions_overlay', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_packages_outdated', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_packages_overlay', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_versions_upstream', self.gf('django.db.models.fields.IntegerField')(default=0)),
('herd', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['euscan.Herd'])),
('n_versions_gentoo', self.gf('django.db.models.fields.IntegerField')(default=0)),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('datetime', self.gf('django.db.models.fields.DateTimeField')()),
))
db.send_create_signal('euscan', ['HerdLog'])
# Adding model 'MaintainerLog'
db.create_table('euscan_maintainerlog', (
('n_packages_gentoo', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_versions_overlay', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_packages_outdated', self.gf('django.db.models.fields.IntegerField')(default=0)),
('maintainer', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['euscan.Maintainer'])),
('n_packages_overlay', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_versions_upstream', self.gf('django.db.models.fields.IntegerField')(default=0)),
('n_versions_gentoo', self.gf('django.db.models.fields.IntegerField')(default=0)),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('datetime', self.gf('django.db.models.fields.DateTimeField')()),
))
db.send_create_signal('euscan', ['MaintainerLog'])
# Deleting model 'Log'
db.delete_table('euscan_log')
models = {
'euscan.euscanresult': {
'Meta': {'object_name': 'EuscanResult'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'euscan.herd': {
'Meta': {'object_name': 'Herd'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'euscan.log': {
'Meta': {'object_name': 'Log'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'euscan.maintainer': {
'Meta': {'object_name': 'Maintainer'},
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'euscan.package': {
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
'homepage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'euscan.version': {
'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
}
}
complete_apps = ['euscan']

130
euscanwww/euscan/migrations/0006_auto__add_worldlog__add_categorylog__add_herdlog__add_maintainerlog.py Normal file
View File

@ -0,0 +1,130 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'WorldLog'
db.create_table('euscan_worldlog', (
('log_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['euscan.Log'], unique=True, primary_key=True)),
))
db.send_create_signal('euscan', ['WorldLog'])
# Adding model 'CategoryLog'
db.create_table('euscan_categorylog', (
('log_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['euscan.Log'], unique=True, primary_key=True)),
('category', self.gf('django.db.models.fields.CharField')(max_length=128)),
))
db.send_create_signal('euscan', ['CategoryLog'])
# Adding model 'HerdLog'
db.create_table('euscan_herdlog', (
('log_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['euscan.Log'], unique=True, primary_key=True)),
('herd', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['euscan.Herd'])),
))
db.send_create_signal('euscan', ['HerdLog'])
# Adding model 'MaintainerLog'
db.create_table('euscan_maintainerlog', (
('log_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['euscan.Log'], unique=True, primary_key=True)),
('maintainer', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['euscan.Maintainer'])),
))
db.send_create_signal('euscan', ['MaintainerLog'])
def backwards(self, orm):
# Deleting model 'WorldLog'
db.delete_table('euscan_worldlog')
# Deleting model 'CategoryLog'
db.delete_table('euscan_categorylog')
# Deleting model 'HerdLog'
db.delete_table('euscan_herdlog')
# Deleting model 'MaintainerLog'
db.delete_table('euscan_maintainerlog')
models = {
'euscan.categorylog': {
'Meta': {'object_name': 'CategoryLog', '_ormbases': ['euscan.Log']},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'euscan.euscanresult': {
'Meta': {'object_name': 'EuscanResult'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'euscan.herd': {
'Meta': {'object_name': 'Herd'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'euscan.herdlog': {
'Meta': {'object_name': 'HerdLog', '_ormbases': ['euscan.Log']},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Herd']"}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'euscan.log': {
'Meta': {'object_name': 'Log'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'euscan.maintainer': {
'Meta': {'object_name': 'Maintainer'},
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'euscan.maintainerlog': {
'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['euscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Maintainer']"})
},
'euscan.package': {
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
'homepage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'euscan.version': {
'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'euscan.worldlog': {
'Meta': {'object_name': 'WorldLog', '_ormbases': ['euscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'})
}
}
complete_apps = ['euscan']

47
euscanwww/euscan/models.py
View File

@ -58,8 +58,7 @@ class EuscanResult(models.Model):
result = models.TextField(blank=True)
# Keep data for charts
class CategoryLog(models.Model):
category = models.CharField(max_length=128)
class Log(models.Model):
datetime = models.DateTimeField()
n_packages_gentoo = models.IntegerField(default=0) # Packages up to date in the main portage tree
@ -71,43 +70,29 @@ class CategoryLog(models.Model):
n_versions_upstream = models.IntegerField(default=0) # Upstream versions, not in the main tree or overlays
def __unicode__(self):
return u'%s [%d:%d:%d] [%d:%d:%d]' % \
(self.category, self.n_packages_gentoo, self.n_packages_overlay, self.n_packages_outdated, \
return u'[%d:%d:%d] [%d:%d:%d]' % \
(self.n_packages_gentoo, self.n_packages_overlay, self.n_packages_outdated, \
self.n_versions_gentoo, self.n_versions_overlay, self.n_versions_upstream)
class WorldLog(Log):
def __unicode__(self):
return u'world ' + Log.__unicode__(self)
class HerdLog(models.Model):
class CategoryLog(Log):
category = models.CharField(max_length=128)
def __unicode__(self):
return u'%s %s' % (self.category, Log.__unicode__(self))
class HerdLog(Log):
herd = models.ForeignKey(Herd)
datetime = models.DateTimeField()
n_packages_gentoo = models.IntegerField(default=0)
n_packages_overlay = models.IntegerField(default=0)
n_packages_outdated = models.IntegerField(default=0)
n_versions_gentoo = models.IntegerField(default=0)
n_versions_overlay = models.IntegerField(default=0)
n_versions_upstream = models.IntegerField(default=0)
def __unicode__(self):
return u'%s [%d:%d:%d] [%d:%d:%d]' % \
(self.herd, self.n_packages_gentoo, self.n_packages_overlay, self.n_packages_outdated, \
self.n_versions_gentoo, self.n_versions_overlay, self.n_versions_upstream)
return u'%s %s' % (self.herd, Log.__unicode__(self))
class MaintainerLog(models.Model):
class MaintainerLog(Log):
maintainer = models.ForeignKey(Maintainer)
datetime = models.DateTimeField()
n_packages_gentoo = models.IntegerField(default=0)
n_packages_overlay = models.IntegerField(default=0)
n_packages_outdated = models.IntegerField(default=0)
n_versions_gentoo = models.IntegerField(default=0)
n_versions_overlay = models.IntegerField(default=0)
n_versions_upstream = models.IntegerField(default=0)
def __unicode__(self):
return u'%s [%d:%d:%d] [%d:%d:%d]' % \
(self.maintainer, self.n_packages_gentoo, self.n_packages_overlay, self.n_packages_outdated, \
self.n_versions_gentoo, self.n_versions_overlay, self.n_versions_upstream)
return u'%s %s' % (self.maintainer, Log.__unicode__(self))

5
euscanwww/scripts/euscan-update.sh
View File

@ -17,7 +17,7 @@
# eix-update
## Scan portage (packages, versions)
# python manage.py scan-portage --all
# python manage.py scan-portage --all --purge
## Scan metadata (herds, maintainers, homepages, ...)
# python manage.py scan-metadata --all
@ -25,3 +25,6 @@
## Scan uptsream packages
# python manage.py scan-upstream --all
# eix --only-names -x | gparallel --jobs 400% euscan | python manage.py scan-upstream --feed
## Update counters
# python manage.py update-counters

2
euscanwww/templates/euscan/category.html
View File

@ -16,7 +16,9 @@
<img src="{% url euscan.views.chart_category category 'pie-packages' %}" />
<h4>All Time</h4>
<img src="{% url euscan.views.chart_category category 'versions-weekly' %}" />
<img src="{% url euscan.views.chart_category category 'versions-monthly' %}" />
<img src="{% url euscan.views.chart_category category 'packages-weekly' %}" />
<img src="{% url euscan.views.chart_category category 'packages-monthly' %}" />
{% endblock %}