euscanwww: add new charts

Signed-off-by: Corentin Chary <corentincj@iksaif.net>

TODO

@@ -1,7 +1,9 @@
 TODO
 ====
 
-- rss feed and or weekly/monthly mail report
+euscan
+------
+
 - respect robots.txt (portscout)
 - check other distros (youri)
 - clean blacklist system
@@ -17,3 +19,10 @@ Site Handlers
 - ftp.kde.org: doesn't scan the "unstable" tree
 - mysql: should use http://downloads.mysql.com/archives/
 - mariadb: should use http://downloads.askmonty.org/MariaDB/+releases/
+
+
+euscanwww
+---------
+
+- rss feed and or weekly/monthly mail report
+

euscanwww/euscan/charts.py

@@ -7,6 +7,8 @@ from django.db.models import F, Sum, Max
 from euscan.models import Version, Package, Herd, Maintainer
 from euscan.models import CategoryLog
 
+import rrdtool
+
 import pylab
 import matplotlib
 
@@ -35,6 +37,20 @@ def chart_alive(name):
         return False
     return True
 
+def rrd_name(**kwargs):
+    name = ""
+
+    if 'category' in kwargs and kwargs['category']:
+        name = 'category-%s' % kwargs['category']
+    elif 'herd' in kwargs and kwargs['herd']:
+        name = 'herd-%d' % kwargs['herd'].id
+    elif 'maintainer' in kwargs and kwargs['maintainer']:
+        name = 'maintainer-%d' % kwargs['maintainer'].id
+    else:
+        name = 'world'
+
+    return name
+
 def chart_name(name, **kwargs):
     if 'category' in kwargs and kwargs['category']:
         name += '-%s' % kwargs['category']
@@ -42,9 +58,14 @@ def chart_name(name, **kwargs):
         name += '-h-%d' % kwargs['herd'].id
     if 'maintainer' in kwargs and kwargs['maintainer']:
         name += '-m-%d' % kwargs['maintainer'].id
+
+    for kw in ('-small', '-weekly', '-monthly', '-yearly'):
+        if kw in kwargs:
+            name += kw
+
     return name + ".png"
 
-def packages(**kwargs):
+def getpackages(**kwargs):
     packages = Package.objects
 
     if 'category' in kwargs and kwargs['category']:
@@ -72,9 +93,9 @@ def cached_pylab_chart(f):
 
 @cached_pylab_chart
 def pie_versions(**kwargs):
-    n_packaged = xint(packages(**kwargs).aggregate(Sum('n_packaged'))['n_packaged__sum'])
+    n_packaged = xint(getpackages(**kwargs).aggregate(Sum('n_packaged'))['n_packaged__sum'])
-    n_overlay = xint(packages(**kwargs).aggregate(Sum('n_overlay'))['n_overlay__sum'])
+    n_overlay = xint(getpackages(**kwargs).aggregate(Sum('n_overlay'))['n_overlay__sum'])
-    n_versions = xint(packages(**kwargs).aggregate(Sum('n_versions'))['n_versions__sum'])
+    n_versions = xint(getpackages(**kwargs).aggregate(Sum('n_versions'))['n_versions__sum'])
     n_upstream = n_versions - n_packaged - n_overlay
 
     pylab.figure(1, figsize=(3.5,3.5))
@@ -93,9 +114,9 @@ def pie_versions(**kwargs):
 
 @cached_pylab_chart
 def pie_packages(**kwargs):
-    n_packages = packages(**kwargs).count()
+    n_packages = getpackages(**kwargs).count()
-    n_packages_uptodate_main = packages(**kwargs).filter(n_versions=F('n_packaged')).count()
+    n_packages_uptodate_main = getpackages(**kwargs).filter(n_versions=F('n_packaged')).count()
-    n_packages_uptodate_all = packages(**kwargs).filter(n_versions=F('n_packaged') + F('n_overlay')).count()
+    n_packages_uptodate_all = getpackages(**kwargs).filter(n_versions=F('n_packaged') + F('n_overlay')).count()
     n_packages_outdated = n_packages - n_packages_uptodate_all
     n_packages_uptodate_ovl = n_packages_uptodate_all - n_packages_uptodate_main
 
@@ -113,3 +134,121 @@ def pie_packages(**kwargs):
     pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', shadow=True)
     pylab.title('Packages', bbox={'facecolor':'0.8', 'pad':5})
 
+
+def rrd_path(name):
+    return str(os.path.join(settings.RRD_ROOT, name + '.rrd'))
+
+def rrd_create(name, start):
+    path = rrd_path(name)
+    if os.path.exists(path):
+        return
+    rrdtool.create(path, '--step', '86400',
+                   '--start', '%s' % int(start - 10),
+                   'DS:n_packages_gentoo:GAUGE:4294967295:0:U',
+                   'DS:n_packages_overlay:GAUGE:4294967295:0:U',
+                   'DS:n_packages_outdated:GAUGE:4294967295:0:U',
+                   'DS:n_versions_gentoo:GAUGE:4294967295:0:U',
+                   'DS:n_versions_overlay:GAUGE:4294967295:0:U',
+                   'DS:n_versions_upstream:GAUGE:4294967295:0:U',
+                   'RRA:AVERAGE:0.5:1:100',
+                   'RRA:AVERAGE:0.5:5:200',
+                   'RRA:AVERAGE:0.5:10:200')
+
+def rrd_update(name, datetime, values):
+    now = time.mktime(datetime.timetuple())
+    rrd_create(name, now)
+    rrdtool.update(rrd_path(name),
+                   '%d:%d:%d:%d:%d:%d:%d' % \
+                       (now, values.n_packages_gentoo, values.n_packages_overlay, values.n_packages_outdated, \
+                            values.n_versions_gentoo, values.n_versions_overlay, values.n_versions_upstream))
+
+
+"""
+[-s|--start time] [-e|--end time] [-S|--step seconds]
+[-t|--title string] [-v|--vertical-label string]
+[-w|--width pixels] [-h|--height pixels] [-j|--only-graph] [-D|--full-size-mode][-u|--upper-limit value] [-l|--lower-limit value]
+[-u|--upper-limit value] [-l|--lower-limit value] [-r|--rigid]
+[-A|--alt-autoscale]
+[-M|--alt-autoscale-max]
+[-J|--alt-autoscale-min]
+"""
+
+def cached_rrd_chart(f):
+    def new_f(*args, **kwds):
+        if 'period' not in kwds:
+            kwds['period'] = '-yearly'
+
+        name = chart_name(f.func_name, **kwds)
+        path = os.path.join(CHARTS_ROOT, name)
+
+        if not chart_alive(name):
+            kwds['title'] = '%s (%s)' % (f.func_name, kwds['period'])
+            kwds['steps'] = kwds['period']
+            kwds['vertical-label'] = f.func_name
+            kwds['rrd'] = rrd_path(rrd_name(**kwds))
+            kwds['path'] = path
+
+            kwds['end'] = 'now'
+
+            if kwds['period'] == '-weekly':
+                kwds['start'] = 'now-4weeks'
+            elif kwds['period'] == '-monthly':
+                kwds['start'] = 'now-12months'
+            else:
+                kwds['start'] = 'now-4years'
+
+            if '-small' in kwds and kwds['-small']:
+                kwds['width'] = '100'
+                kwds['height'] = '30'
+                kwds['graph-mode'] = '--only-graph'
+            else:
+                kwds['width'] = '500'
+                kwds['height'] = '170'
+                kwds['graph-mode'] = '--full-size-mode'
+
+            f(*args, **kwds)
+
+        return name
+
+    new_f.func_name = f.func_name
+    return new_f
+
+@cached_rrd_chart
+def packages(**kwargs):
+    rrdtool.graph(str(kwargs['path']),
+                  '--imgformat', 'PNG',
+                  '--width', kwargs['width'],
+                  '--height', kwargs['height'],
+                  kwargs['graph-mode'],
+                  '--start', kwargs['start'],
+                  '--end', kwargs['end'],
+                  '--vertical-label', kwargs['vertical-label'],
+                  '--title', kwargs['title'],
+                  '--lower-limit', '0',
+                  'DEF:n_packages_gentoo=%s:n_packages_gentoo:AVERAGE' % (kwargs['rrd']),
+                  'DEF:n_packages_overlay=%s:n_packages_overlay:AVERAGE' % (kwargs['rrd']),
+                  'DEF:n_packages_outdated=%s:n_packages_outdated:AVERAGE' % (kwargs['rrd']),
+                  'LINE1.25:n_packages_gentoo#008000:Gentoo',
+                  'LINE1.25:n_packages_overlay#0B17FD:Overlay',
+                  'LINE1.25:n_packages_outdated#FF0000:Outdated')
+
+@cached_rrd_chart
+def versions(**kwargs):
+    rrdtool.graph(str(kwargs['path']),
+                  '--imgformat', 'PNG',
+                  '--width', kwargs['width'],
+                  '--height', kwargs['height'],
+                  kwargs['graph-mode'],
+                  '--start', kwargs['start'],
+                  '--end', kwargs['end'],
+                  '--vertical-label', kwargs['vertical-label'],
+                  '--title', kwargs['title'],
+                  '--lower-limit', '0',
+                  'DEF:n_versions_gentoo=%s:n_versions_gentoo:AVERAGE' % (kwargs['rrd']),
+                  'DEF:n_versions_overlay=%s:n_versions_overlay:AVERAGE' % (kwargs['rrd']),
+                  'DEF:n_versions_outdated=%s:n_versions_upstream:AVERAGE' % (kwargs['rrd']),
+                  'LINE1.25:n_versions_gentoo#008000:Gentoo',
+                  'LINE1.25:n_versions_overlay#0B17FD:Overlay',
+                  'LINE1.25:n_versions_outdated#FF0000:Outdated')
+
+

euscanwww/euscan/management/commands/regen-rrds.py

@@ -0,0 +1,21 @@
+import datetime
+
+from optparse import make_option
+
+from django.core.management.base import BaseCommand, CommandError
+from euscanwww.euscan.models import Package, HerdLog, MaintainerLog, CategoryLog, Herd, Maintainer, Version
+from euscanwww.euscan import charts
+
+class Command(BaseCommand):
+    _overlays = {}
+    help = 'Regenerate rrd database'
+
+    def handle(self, *args, **options):
+        for clog in CategoryLog.objects.all():
+            charts.rrd_update('category-%s' % clog.category, clog.datetime, clog)
+
+        for hlog in HerdLog.objects.all():
+            charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
+
+        for mlog in MaintainerLog.objects.all():
+            charts.rrd_update('maintainer-%d' % mlog.maintainer.id, mlog.datetime, mlog)

euscanwww/euscan/management/commands/update-counters.py

@@ -6,6 +6,15 @@ from django.db.models import Count, Sum
 from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand, CommandError
 from euscanwww.euscan.models import Package, HerdLog, MaintainerLog, CategoryLog, Herd, Maintainer, Version
+from euscanwww.euscan import charts
+
+class World:
+    n_packages_gentoo   = 0
+    n_packages_overlay  = 0
+    n_packages_outdated = 0
+    n_versions_gentoo   = 0
+    n_versions_overlay  = 0
+    n_versions_upstream = 0
 
 class Command(BaseCommand):
     _overlays = {}
@@ -26,6 +35,7 @@ class Command(BaseCommand):
         categories = {}
         herds = {}
         maintainers = {}
+        world = World()
 
         # Could be done using raw SQL queries, but I don't have time for that
         # right now ...
@@ -55,40 +65,60 @@ class Command(BaseCommand):
             package.n_overlay = Version.objects.filter(package=package, packaged=True).exclude(overlay='gentoo').count()
             package.save()
 
+            n_packages_gentoo = int(package.n_packaged == package.n_versions)
+            n_packages_overlay = int(package.n_overlay and package.n_packaged + package.n_overlay == package.n_versions)
+            n_packages_outdated = int(package.n_packaged + package.n_overlay < package.n_versions)
+
             for herd in package.herds.all():
-                herds[herd].n_packages += 1
+                herds[herd].n_packages_gentoo   += n_packages_gentoo
-                herds[herd].n_versions += package.n_versions
+                herds[herd].n_packages_overlay  += n_packages_overlay
-                herds[herd].n_packaged += package.n_packaged
+                herds[herd].n_packages_outdated += n_packages_outdated
-                herds[herd].n_overlay += package.n_overlay
+
+                herds[herd].n_versions_gentoo   += package.n_packaged
+                herds[herd].n_versions_overlay  += package.n_overlay
+                herds[herd].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
 
             for maintainer in package.maintainers.all():
-                maintainers[maintainer].n_packages += 1
+                maintainers[maintainer].n_packages_gentoo   += n_packages_gentoo
-                maintainers[maintainer].n_versions += package.n_versions
+                maintainers[maintainer].n_packages_overlay  += n_packages_overlay
-                maintainers[maintainer].n_packaged += package.n_packaged
+                maintainers[maintainer].n_packages_outdated += n_packages_outdated
-                maintainers[maintainer].n_overlay += package.n_overlay
 
-            categories[package.category].n_packages += 1
+                maintainers[maintainer].n_versions_gentoo   += package.n_packaged
-            categories[package.category].n_versions += package.n_versions
+                maintainers[maintainer].n_versions_overlay  += package.n_overlay
-            categories[package.category].n_packaged += package.n_packaged
+                maintainers[maintainer].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
-            categories[package.category].n_overlay += package.n_overlay
+
+            categories[package.category].n_packages_gentoo   += n_packages_gentoo
+            categories[package.category].n_packages_overlay  += n_packages_overlay
+            categories[package.category].n_packages_outdated += n_packages_outdated
+
+            categories[package.category].n_versions_gentoo   += package.n_packaged
+            categories[package.category].n_versions_overlay  += package.n_overlay
+            categories[package.category].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
+
+            world.n_packages_gentoo   += n_packages_gentoo
+            world.n_packages_overlay  += n_packages_overlay
+            world.n_packages_outdated += n_packages_outdated
+
+            world.n_versions_gentoo   += package.n_packaged
+            world.n_versions_overlay  += package.n_overlay
+            world.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
 
         for clog in categories.values():
             if not options['quiet']:
-                self.stdout.write('[c] %s - [%d, %d/%d/%d]\n' %
+                self.stdout.write('[c] %s\n' % clog)
-                                  (clog.category, clog.n_packages,
+            charts.rrd_update('category-%s' % clog.category, now, clog)
-                                   clog.n_packaged, clog.n_overlay, clog.n_versions))
             clog.save()
 
         for hlog in herds.values():
             if not options['quiet']:
-                self.stdout.write('[h] %s - [%d, %d/%d/%d]\n' %
+                self.stdout.write('[h] %s\n' % hlog)
-                                  (hlog.herd, hlog.n_packages,
+            charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
-                                   hlog.n_packaged, hlog.n_overlay, hlog.n_versions))
             hlog.save()
 
         for mlog in maintainers.values():
             if not options['quiet']:
-                self.stdout.write('[m] %s - [%d, %d/%d/%d]\n' %
+                self.stdout.write('[m] %s\n' % mlog)
-                                  (mlog.maintainer, mlog.n_packages,
+            charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
-                                   mlog.n_packaged, mlog.n_overlay, mlog.n_versions))
             mlog.save()
+
+        charts.rrd_update('world', now, world)

euscanwww/euscan/migrations/0004_auto__del_field_categorylog_n_versions__del_field_categorylog_n_packag.py

@@ -0,0 +1,277 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Deleting field 'CategoryLog.n_versions'
+        db.delete_column('euscan_categorylog', 'n_versions')
+
+        # Deleting field 'CategoryLog.n_packaged'
+        db.delete_column('euscan_categorylog', 'n_packaged')
+
+        # Deleting field 'CategoryLog.n_packages'
+        db.delete_column('euscan_categorylog', 'n_packages')
+
+        # Deleting field 'CategoryLog.n_overlay'
+        db.delete_column('euscan_categorylog', 'n_overlay')
+
+        # Adding field 'CategoryLog.n_packages_gentoo'
+        db.add_column('euscan_categorylog', 'n_packages_gentoo', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'CategoryLog.n_packages_overlay'
+        db.add_column('euscan_categorylog', 'n_packages_overlay', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'CategoryLog.n_packages_outdated'
+        db.add_column('euscan_categorylog', 'n_packages_outdated', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'CategoryLog.n_versions_gentoo'
+        db.add_column('euscan_categorylog', 'n_versions_gentoo', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'CategoryLog.n_versions_overlay'
+        db.add_column('euscan_categorylog', 'n_versions_overlay', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'CategoryLog.n_versions_upstream'
+        db.add_column('euscan_categorylog', 'n_versions_upstream', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Deleting field 'HerdLog.n_versions'
+        db.delete_column('euscan_herdlog', 'n_versions')
+
+        # Deleting field 'HerdLog.n_packaged'
+        db.delete_column('euscan_herdlog', 'n_packaged')
+
+        # Deleting field 'HerdLog.n_packages'
+        db.delete_column('euscan_herdlog', 'n_packages')
+
+        # Deleting field 'HerdLog.n_overlay'
+        db.delete_column('euscan_herdlog', 'n_overlay')
+
+        # Adding field 'HerdLog.n_packages_gentoo'
+        db.add_column('euscan_herdlog', 'n_packages_gentoo', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'HerdLog.n_packages_overlay'
+        db.add_column('euscan_herdlog', 'n_packages_overlay', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'HerdLog.n_packages_outdated'
+        db.add_column('euscan_herdlog', 'n_packages_outdated', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'HerdLog.n_versions_gentoo'
+        db.add_column('euscan_herdlog', 'n_versions_gentoo', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'HerdLog.n_versions_overlay'
+        db.add_column('euscan_herdlog', 'n_versions_overlay', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'HerdLog.n_versions_upstream'
+        db.add_column('euscan_herdlog', 'n_versions_upstream', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Deleting field 'MaintainerLog.n_versions'
+        db.delete_column('euscan_maintainerlog', 'n_versions')
+
+        # Deleting field 'MaintainerLog.n_packaged'
+        db.delete_column('euscan_maintainerlog', 'n_packaged')
+
+        # Deleting field 'MaintainerLog.n_packages'
+        db.delete_column('euscan_maintainerlog', 'n_packages')
+
+        # Deleting field 'MaintainerLog.n_overlay'
+        db.delete_column('euscan_maintainerlog', 'n_overlay')
+
+        # Adding field 'MaintainerLog.n_packages_gentoo'
+        db.add_column('euscan_maintainerlog', 'n_packages_gentoo', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'MaintainerLog.n_packages_overlay'
+        db.add_column('euscan_maintainerlog', 'n_packages_overlay', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'MaintainerLog.n_packages_outdated'
+        db.add_column('euscan_maintainerlog', 'n_packages_outdated', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'MaintainerLog.n_versions_gentoo'
+        db.add_column('euscan_maintainerlog', 'n_versions_gentoo', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'MaintainerLog.n_versions_overlay'
+        db.add_column('euscan_maintainerlog', 'n_versions_overlay', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'MaintainerLog.n_versions_upstream'
+        db.add_column('euscan_maintainerlog', 'n_versions_upstream', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+
+    def backwards(self, orm):
+        
+        # Adding field 'CategoryLog.n_versions'
+        db.add_column('euscan_categorylog', 'n_versions', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'CategoryLog.n_packaged'
+        db.add_column('euscan_categorylog', 'n_packaged', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'CategoryLog.n_packages'
+        db.add_column('euscan_categorylog', 'n_packages', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'CategoryLog.n_overlay'
+        db.add_column('euscan_categorylog', 'n_overlay', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Deleting field 'CategoryLog.n_packages_gentoo'
+        db.delete_column('euscan_categorylog', 'n_packages_gentoo')
+
+        # Deleting field 'CategoryLog.n_packages_overlay'
+        db.delete_column('euscan_categorylog', 'n_packages_overlay')
+
+        # Deleting field 'CategoryLog.n_packages_outdated'
+        db.delete_column('euscan_categorylog', 'n_packages_outdated')
+
+        # Deleting field 'CategoryLog.n_versions_gentoo'
+        db.delete_column('euscan_categorylog', 'n_versions_gentoo')
+
+        # Deleting field 'CategoryLog.n_versions_overlay'
+        db.delete_column('euscan_categorylog', 'n_versions_overlay')
+
+        # Deleting field 'CategoryLog.n_versions_upstream'
+        db.delete_column('euscan_categorylog', 'n_versions_upstream')
+
+        # Adding field 'HerdLog.n_versions'
+        db.add_column('euscan_herdlog', 'n_versions', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'HerdLog.n_packaged'
+        db.add_column('euscan_herdlog', 'n_packaged', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'HerdLog.n_packages'
+        db.add_column('euscan_herdlog', 'n_packages', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'HerdLog.n_overlay'
+        db.add_column('euscan_herdlog', 'n_overlay', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Deleting field 'HerdLog.n_packages_gentoo'
+        db.delete_column('euscan_herdlog', 'n_packages_gentoo')
+
+        # Deleting field 'HerdLog.n_packages_overlay'
+        db.delete_column('euscan_herdlog', 'n_packages_overlay')
+
+        # Deleting field 'HerdLog.n_packages_outdated'
+        db.delete_column('euscan_herdlog', 'n_packages_outdated')
+
+        # Deleting field 'HerdLog.n_versions_gentoo'
+        db.delete_column('euscan_herdlog', 'n_versions_gentoo')
+
+        # Deleting field 'HerdLog.n_versions_overlay'
+        db.delete_column('euscan_herdlog', 'n_versions_overlay')
+
+        # Deleting field 'HerdLog.n_versions_upstream'
+        db.delete_column('euscan_herdlog', 'n_versions_upstream')
+
+        # Adding field 'MaintainerLog.n_versions'
+        db.add_column('euscan_maintainerlog', 'n_versions', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'MaintainerLog.n_packaged'
+        db.add_column('euscan_maintainerlog', 'n_packaged', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'MaintainerLog.n_packages'
+        db.add_column('euscan_maintainerlog', 'n_packages', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Adding field 'MaintainerLog.n_overlay'
+        db.add_column('euscan_maintainerlog', 'n_overlay', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+        # Deleting field 'MaintainerLog.n_packages_gentoo'
+        db.delete_column('euscan_maintainerlog', 'n_packages_gentoo')
+
+        # Deleting field 'MaintainerLog.n_packages_overlay'
+        db.delete_column('euscan_maintainerlog', 'n_packages_overlay')
+
+        # Deleting field 'MaintainerLog.n_packages_outdated'
+        db.delete_column('euscan_maintainerlog', 'n_packages_outdated')
+
+        # Deleting field 'MaintainerLog.n_versions_gentoo'
+        db.delete_column('euscan_maintainerlog', 'n_versions_gentoo')
+
+        # Deleting field 'MaintainerLog.n_versions_overlay'
+        db.delete_column('euscan_maintainerlog', 'n_versions_overlay')
+
+        # Deleting field 'MaintainerLog.n_versions_upstream'
+        db.delete_column('euscan_maintainerlog', 'n_versions_upstream')
+
+
+    models = {
+        'euscan.categorylog': {
+            'Meta': {'object_name': 'CategoryLog'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'euscan.euscanresult': {
+            'Meta': {'object_name': 'EuscanResult'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
+            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'euscan.herd': {
+            'Meta': {'object_name': 'Herd'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
+            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'euscan.herdlog': {
+            'Meta': {'object_name': 'HerdLog'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Herd']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'euscan.maintainer': {
+            'Meta': {'object_name': 'Maintainer'},
+            'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'euscan.maintainerlog': {
+            'Meta': {'object_name': 'MaintainerLog'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Maintainer']"}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'euscan.package': {
+            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
+            'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'euscan.version': {
+            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        }
+    }
+
+    complete_apps = ['euscan']

euscanwww/euscan/models.py

@@ -62,25 +62,52 @@ class CategoryLog(models.Model):
     category = models.CharField(max_length=128)
     datetime = models.DateTimeField()
 
-    n_packages = models.IntegerField(default=0)
+    n_packages_gentoo   = models.IntegerField(default=0) # Packages up to date in the main portage tree
-    n_versions = models.IntegerField(default=0)
+    n_packages_overlay  = models.IntegerField(default=0) # Packages up to date in an overlay
-    n_packaged = models.IntegerField(default=0)
+    n_packages_outdated = models.IntegerField(default=0) # Packages outdated
-    n_overlay = models.IntegerField(default=0)
+
+    n_versions_gentoo   = models.IntegerField(default=0) # Versions in the main portage tree
+    n_versions_overlay  = models.IntegerField(default=0) # Versions in overlays
+    n_versions_upstream = models.IntegerField(default=0) # Upstream versions, not in the main tree or overlays
+
+    def __unicode__(self):
+        return u'%s [%d:%d:%d] [%d:%d:%d]' % \
+            (self.category, self.n_packages_gentoo, self.n_packages_overlay, self.n_packages_outdated, \
+                 self.n_versions_gentoo, self.n_versions_overlay, self.n_versions_upstream)
+
 
 class HerdLog(models.Model):
     herd = models.ForeignKey(Herd)
     datetime = models.DateTimeField()
 
-    n_packages = models.IntegerField(default=0)
+    n_packages_gentoo   = models.IntegerField(default=0)
-    n_versions = models.IntegerField(default=0)
+    n_packages_overlay  = models.IntegerField(default=0)
-    n_packaged = models.IntegerField(default=0)
+    n_packages_outdated = models.IntegerField(default=0)
-    n_overlay = models.IntegerField(default=0)
+
+    n_versions_gentoo   = models.IntegerField(default=0)
+    n_versions_overlay  = models.IntegerField(default=0)
+    n_versions_upstream = models.IntegerField(default=0)
+
+    def __unicode__(self):
+        return u'%s [%d:%d:%d] [%d:%d:%d]' % \
+            (self.herd, self.n_packages_gentoo, self.n_packages_overlay, self.n_packages_outdated, \
+                 self.n_versions_gentoo, self.n_versions_overlay, self.n_versions_upstream)
+
 
 class MaintainerLog(models.Model):
     maintainer = models.ForeignKey(Maintainer)
     datetime = models.DateTimeField()
 
-    n_packages = models.IntegerField(default=0)
+    n_packages_gentoo   = models.IntegerField(default=0)
-    n_versions = models.IntegerField(default=0)
+    n_packages_overlay  = models.IntegerField(default=0)
-    n_packaged = models.IntegerField(default=0)
+    n_packages_outdated = models.IntegerField(default=0)
-    n_overlay = models.IntegerField(default=0)
+
+    n_versions_gentoo   = models.IntegerField(default=0)
+    n_versions_overlay  = models.IntegerField(default=0)
+    n_versions_upstream = models.IntegerField(default=0)
+
+    def __unicode__(self):
+        return u'%s [%d:%d:%d] [%d:%d:%d]' % \
+            (self.maintainer, self.n_packages_gentoo, self.n_packages_overlay, self.n_packages_outdated, \
+                 self.n_versions_gentoo, self.n_versions_overlay, self.n_versions_upstream)
+

euscanwww/euscan/views.py

@@ -138,10 +138,21 @@ def chart(request, **kwargs):
     if 'herd' in kwargs:
         kwargs['herd'] = get_object_or_404(Herd, herd=kwargs['herd'])
 
+    for kw in ('-small', '-weekly', '-monthly', '-yearly'):
+        if chart.endswith(kw):
+            if kw in ('-weekly', '-monthly', '-yearly'):
+                kwargs['period'] = kw
+            kwargs[kw] = True
+            chart = chart[:-len(kw)]
+
     if chart == 'pie-packages':
         path = charts.pie_packages(**kwargs)
     elif chart == 'pie-versions':
         path = charts.pie_versions(**kwargs)
+    elif chart == 'packages':
+        path = charts.packages(**kwargs)
+    elif chart == 'versions':
+        path = charts.versions(**kwargs)
     else:
         raise Http404()
 

euscanwww/settings.py

@@ -31,6 +31,8 @@ DATABASES = {
     }
 }
 
+RRD_ROOT = os.path.join(os.path.dirname( __file__ ), 'rrd')
+
 # Local time zone for this installation. Choices can be found here:
 # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
 # although not all choices may be available on all operating systems.

euscanwww/templates/euscan/category.html

@@ -14,4 +14,9 @@
 <h4>Current statistics</h4>
 <img src="{% url euscan.views.chart_category category 'pie-versions' %}" />
 <img src="{% url euscan.views.chart_category category 'pie-packages' %}" />
+
+<h4>All Time</h4>
+<img src="{% url euscan.views.chart_category category 'versions-monthly' %}" />
+<img src="{% url euscan.views.chart_category category 'packages-monthly' %}" />
+
 {% endblock %}

euscanwww/templates/euscan/herd.html

@@ -14,4 +14,9 @@
 <h4>Current statistics</h4>
 <img src="{% url euscan.views.chart_herd herd.herd 'pie-versions' %}" />
 <img src="{% url euscan.views.chart_herd herd.herd 'pie-packages' %}" />
+
+<h4>All Time</h4>
+<img src="{% url euscan.views.chart_herd herd.herd 'versions-monthly' %}" />
+<img src="{% url euscan.views.chart_herd herd.herd 'packages-monthly' %}" />
+
 {% endblock %}

euscanwww/templates/euscan/maintainer.html

@@ -15,4 +15,8 @@
 <img src="{% url euscan.views.chart_maintainer maintainer.id 'pie-versions' %}" />
 <img src="{% url euscan.views.chart_maintainer maintainer.id 'pie-packages' %}" />
 
+<h4>All Time</h4>
+<img src="{% url euscan.views.chart_maintainer maintainer.id 'versions-monthly' %}" />
+<img src="{% url euscan.views.chart_maintainer maintainer.id 'packages-monthly' %}" />
+
 {% endblock %}

euscanwww/templates/euscan/statistics.html

@@ -10,4 +10,10 @@
 
 <h3>All time statistics</h3>
 
+<img src="{% url euscan.views.chart 'packages-weekly' %}" />
+<img src="{% url euscan.views.chart 'packages-monthly' %}" />
+
+<img src="{% url euscan.views.chart 'versions-weekly' %}" />
+<img src="{% url euscan.views.chart 'versions-monthly' %}" />
+
 {% endblock %}