euscanwww: add freshness and last versions

Signed-off-by: Corentin Chary <corentincj@iksaif.net>

euscanwww/api/handlers.py

@@ -108,7 +108,10 @@ class CategoriesHandler(AnonymousBaseHandler):
 # /api/1.0/packages/by-herd/
 class PackagesHandler(AnonymousBaseHandler):
     allowed_methods = ('GET',)
-    fields = ('category', 'name', 'n_packaged', 'n_overlay', 'n_versions')
+    fields = ('category', 'name', 'n_packaged', 'n_overlay', 'n_versions',
+              ('last_version_gentoo', ('version',)),
+              ('last_version_overlay', ('version',)),
+              ('last_version_upstream', ('version',)))
     model = Package
 
     @catch_and_return(ObjectDoesNotExist, rc.NOT_FOUND)
@@ -117,15 +120,18 @@ class PackagesHandler(AnonymousBaseHandler):
 
         if 'category' in kwargs:
             packages = Package.objects.filter(category=kwargs['category'])
-            data = { 'category' : kwargs['category'], 'packages' : packages }
+            data = { 'category' : kwargs['category'] }
         elif 'herd' in kwargs:
             herd = Herd.objects.get(herd=kwargs['herd'])
             packages = Package.objects.filter(herds__id=herd.id)
-            data = { 'herd' : herd, 'packages' : packages }
+            data = { 'herd' : herd }
         elif 'maintainer_id' in kwargs:
             maintainer = Maintainer.objects.get(id=kwargs['maintainer_id'])
             packages = Package.objects.filter(maintainers__id=maintainer.id)
-            data = { 'maintainer' : maintainer, 'packages' : packages }
+            data = { 'maintainer' : maintainer }
+
+        packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
+        data['packages'] = packages
 
         if not data:
             return rc.NOT_FOUND

euscanwww/euscan/management/commands/update-counters.py

@@ -5,6 +5,7 @@ from optparse import make_option
 from django.db.models import Count, Sum
 from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand, CommandError
+
 from euscanwww.euscan.models import Package, Herd, Maintainer, Version
 from euscanwww.euscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
 from euscanwww.euscan import charts
@@ -19,6 +20,17 @@ class Command(BaseCommand):
             dest='quiet',
             default=False,
             help='Be quiet'),
+        make_option('--fast',
+            action='store_true',
+            dest='fast',
+            default=False,
+            help='Skip sanity checks'),
+        make_option('--nolog',
+            action='store_true',
+            dest='nolog',
+            default=False,
+            help='Skip logs'),
+
         )
 
     @commit_on_success
@@ -29,76 +41,118 @@ class Command(BaseCommand):
         herds = {}
         maintainers = {}
 
-        '''
+        wlog = None
-        Could be done using raw SQL queries, but I don't have time for that
-        right now ...
-        '''
 
-        wlog = WorldLog()
+        if not options['nolog']:
-        wlog.datetime = now
+            wlog = WorldLog()
+            wlog.datetime = now
 
-        for cat in Package.objects.values('category').distinct():
+            for cat in Package.objects.values('category').distinct():
-            clog = CategoryLog()
+                clog = CategoryLog()
-            clog.datetime = now
+                clog.datetime = now
-            clog.category = cat['category']
+                clog.category = cat['category']
-            categories[clog.category] = clog
+                categories[clog.category] = clog
 
-        for herd in Herd.objects.all():
+            for herd in Herd.objects.all():
-            hlog = HerdLog()
+                hlog = HerdLog()
-            hlog.datetime = now
+                hlog.datetime = now
-            hlog.herd = herd
+                hlog.herd = herd
-            herds[herd] = hlog
+                herds[herd.id] = hlog
 
-        for maintainer in Maintainer.objects.all():
+            for maintainer in Maintainer.objects.all():
-            mlog = MaintainerLog()
+                mlog = MaintainerLog()
-            mlog.datetime = now
+                mlog.datetime = now
-            mlog.maintainer = maintainer
+                mlog.maintainer = maintainer
-            maintainers[maintainer] = mlog
+                maintainers[maintainer.id] = mlog
 
-        for package in Package.objects.all():
+        package_queryset = Package.objects.all()
-            # Should not be needed, but can't hurt
+
-            package.n_versions = Version.objects.filter(package=package).count()
+        n_versions = {}
-            package.n_packaged = Version.objects.filter(package=package, packaged=True, overlay='gentoo').count()
+        n_packaged = {}
-            package.n_overlay = Version.objects.filter(package=package, packaged=True).exclude(overlay='gentoo').count()
+        n_overlay = {}
-            package.save()
+
+        last_versions_gentoo = {}
+        last_versions_overlay = {}
+        last_versions_upstream = {}
+
+        def add_safe(storage, key):
+            if key not in storage:
+                storage[key] = 1
+            else:
+                storage[key] += 1
+
+        def add_last_ver(storage, version):
+            key = version['package_id']
+            if key not in storage:
+                storage[key] = version
+                return
+            if version['version'].startswith('9999'):
+                return
+            if storage[key]['version'] < version['version']:
+                storage[key] = version
+
+        if not options['fast']:
+            attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
+            for version in Version.objects.all().values(*attrs):
+                overlay, packaged = version['overlay'], version['packaged']
+                package_id = version['package_id']
+
+                add_safe(n_versions, package_id)
+
+                if not packaged:
+                    add_last_ver(last_versions_upstream, version)
+                    continue
+                if overlay == 'gentoo':
+                    add_safe(n_packaged, package_id)
+                    add_last_ver(last_versions_gentoo, version)
+                else:
+                    add_safe(n_overlay, package_id)
+                    add_last_ver(last_versions_overlay, version)
+
+        for package in package_queryset.select_related('herds', 'maintainers'):
+            if not options['fast']:
+                package.n_versions = n_versions.get(package.id, 0)
+                package.n_packaged = n_packaged.get(package.id, 0)
+                package.n_overlay = n_overlay.get(package.id, 0)
+
+                default = {'id' : -1}
+                package.last_version_gentoo_id = last_versions_gentoo.get(package.id, default)['id']
+                package.last_version_overlay_id = last_versions_overlay.get(package.id, default)['id']
+                package.last_version_upstream_id = last_versions_upstream.get(package.id, default)['id']
+
+                package.save()
 
             n_packages_gentoo = int(package.n_packaged == package.n_versions)
             n_packages_overlay = int(package.n_overlay and package.n_packaged + package.n_overlay == package.n_versions)
             n_packages_outdated = int(package.n_packaged + package.n_overlay < package.n_versions)
 
-            for herd in package.herds.all():
+            def update_row(storage, key):
-                herds[herd].n_packages_gentoo   += n_packages_gentoo
+                storage[key].n_packages_gentoo   += n_packages_gentoo
-                herds[herd].n_packages_overlay  += n_packages_overlay
+                storage[key].n_packages_overlay  += n_packages_overlay
-                herds[herd].n_packages_outdated += n_packages_outdated
+                storage[key].n_packages_outdated += n_packages_outdated
 
-                herds[herd].n_versions_gentoo   += package.n_packaged
+                storage[key].n_versions_gentoo   += package.n_packaged
-                herds[herd].n_versions_overlay  += package.n_overlay
+                storage[key].n_versions_overlay  += package.n_overlay
-                herds[herd].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
+                storage[key].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
+            def update_log(storage, qs):
+                for row in qs:
+                    update_row(storage, row['id'])
 
-            for maintainer in package.maintainers.all():
+            if not options['nolog']:
-                maintainers[maintainer].n_packages_gentoo   += n_packages_gentoo
+                update_log(herds, package.herds.all().values('id'))
-                maintainers[maintainer].n_packages_overlay  += n_packages_overlay
+                update_log(maintainers, package.maintainers.all().values('id'))
-                maintainers[maintainer].n_packages_outdated += n_packages_outdated
+                update_row(categories, package.category)
 
-                maintainers[maintainer].n_versions_gentoo   += package.n_packaged
+                wlog.n_packages_gentoo   += n_packages_gentoo
-                maintainers[maintainer].n_versions_overlay  += package.n_overlay
+                wlog.n_packages_overlay  += n_packages_overlay
-                maintainers[maintainer].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
+                wlog.n_packages_outdated += n_packages_outdated
 
-            categories[package.category].n_packages_gentoo   += n_packages_gentoo
+                wlog.n_versions_gentoo   += package.n_packaged
-            categories[package.category].n_packages_overlay  += n_packages_overlay
+                wlog.n_versions_overlay  += package.n_overlay
-            categories[package.category].n_packages_outdated += n_packages_outdated
+                wlog.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
 
-            categories[package.category].n_versions_gentoo   += package.n_packaged
+        if options['nolog']:
-            categories[package.category].n_versions_overlay  += package.n_overlay
+            return
-            categories[package.category].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
-
-            wlog.n_packages_gentoo   += n_packages_gentoo
-            wlog.n_packages_overlay  += n_packages_overlay
-            wlog.n_packages_outdated += n_packages_outdated
-
-            wlog.n_versions_gentoo   += package.n_packaged
-            wlog.n_versions_overlay  += package.n_overlay
-            wlog.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
 
         for clog in categories.values():
             if not options['quiet']:
@@ -118,6 +172,5 @@ class Command(BaseCommand):
             charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
             mlog.save()
 
-        wlog.save()
-
         charts.rrd_update('world', now, wlog)
+        wlog.save()

euscanwww/euscan/migrations/0009_auto__add_field_package_last_version_gentoo__add_field_package_last_ve.py

@@ -0,0 +1,125 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding field 'Package.last_version_gentoo'
+        db.add_column('euscan_package', 'last_version_gentoo', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='last_version_gentoo', null=True, to=orm['euscan.Version']), keep_default=False)
+
+        # Adding field 'Package.last_version_overlay'
+        db.add_column('euscan_package', 'last_version_overlay', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='last_version_overlay', null=True, to=orm['euscan.Version']), keep_default=False)
+
+        # Adding field 'Package.last_version_upstream'
+        db.add_column('euscan_package', 'last_version_upstream', self.gf('django.db.models.fields.related.ForeignKey')(blank=True, related_name='last_version_upstream', null=True, to=orm['euscan.Version']), keep_default=False)
+
+
+    def backwards(self, orm):
+        
+        # Deleting field 'Package.last_version_gentoo'
+        db.delete_column('euscan_package', 'last_version_gentoo_id')
+
+        # Deleting field 'Package.last_version_overlay'
+        db.delete_column('euscan_package', 'last_version_overlay_id')
+
+        # Deleting field 'Package.last_version_upstream'
+        db.delete_column('euscan_package', 'last_version_upstream_id')
+
+
+    models = {
+        'euscan.categorylog': {
+            'Meta': {'object_name': 'CategoryLog', '_ormbases': ['euscan.Log']},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'euscan.euscanresult': {
+            'Meta': {'object_name': 'EuscanResult'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
+            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'euscan.herd': {
+            'Meta': {'object_name': 'Herd'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
+            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'euscan.herdlog': {
+            'Meta': {'object_name': 'HerdLog', '_ormbases': ['euscan.Log']},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Herd']"}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'euscan.log': {
+            'Meta': {'object_name': 'Log'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'euscan.maintainer': {
+            'Meta': {'object_name': 'Maintainer'},
+            'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'euscan.maintainerlog': {
+            'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['euscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Maintainer']"})
+        },
+        'euscan.package': {
+            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'to': "orm['euscan.Version']"}),
+            'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'to': "orm['euscan.Version']"}),
+            'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'to': "orm['euscan.Version']"}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
+            'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'euscan.version': {
+            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
+            'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'euscan.versionlog': {
+            'Meta': {'object_name': 'VersionLog'},
+            'action': ('django.db.models.fields.IntegerField', [], {}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2012, 3, 2, 16, 32, 3, 656974)'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'euscan.worldlog': {
+            'Meta': {'object_name': 'WorldLog', '_ormbases': ['euscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        }
+    }
+
+    complete_apps = ['euscan']

euscanwww/euscan/models.py

@@ -30,6 +30,14 @@ class Package(models.Model):
     n_packaged = models.IntegerField(default=0)
     n_overlay = models.IntegerField(default=0)
 
+    ' And we also pre-compute last versions '
+    last_version_gentoo = models.ForeignKey('Version', blank=True, null=True,
+                                            related_name="last_version_gentoo")
+    last_version_overlay = models.ForeignKey('Version', blank=True, null=True,
+                                             related_name="last_version_overlay")
+    last_version_upstream = models.ForeignKey('Version', blank=True, null=True,
+                                              related_name="last_version_upstream")
+
     def __unicode__(self):
         return '%s/%s' % (self.category, self.name)
 

euscanwww/euscan/templatetags/div.py

@@ -3,6 +3,6 @@ from django import template
 register = template.Library()
 
 def div(value, arg=None):
-    return value/arg
+    return float(value)/float(arg)
 
 register.filter('div', div)

euscanwww/euscan/views.py

@@ -37,7 +37,8 @@ def categories(request):
 
 @render_to('euscan/category.html')
 def category(request, category):
-    packages = Package.objects.filter(category=category)
+    packages = Package.objects.filter(category=category).select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
+    print dir(packages[0])
     if not packages:
         raise Http404
     return { 'category' : category, 'packages' : packages }
@@ -94,8 +95,8 @@ def overlay(request, overlay):
 def package(request, category, package):
     package = get_object_or_404(Package, category=category, name=package)
     package.homepages = package.homepage.split(' ')
-    packaged = Version.objects.filter(package=package, packaged=True)
+    packaged = Version.objects.filter(package=package, packaged=True).order_by('version', 'revision')
-    upstream = Version.objects.filter(package=package, packaged=False)
+    upstream = Version.objects.filter(package=package, packaged=False).order_by('version', 'revision')
     log = EuscanResult.objects.filter(package=package).order_by('-datetime')[:1]
     log = log[0] if log else None
     vlog = VersionLog.objects.filter(package=package).order_by('-id')

euscanwww/templates/euscan/_package_cols.html

@@ -4,6 +4,7 @@
 {% load div %}
 
 <td>{{ infos.n_packaged }}</td>
+
 {% if infos.n_overlay == 0 or infos.n_overlay <= infos.n_packaged %}
 <td>
 {% else %}{% if infos.n_overlay < infos.n_packaged %}
@@ -13,6 +14,7 @@
 {% endif %}{% endif %}
   {{ infos.n_overlay }}
 </td>
+
 {% if infos.n_versions == infos.n_packaged|add:infos.n_overlay %}
 <td>
 {% else %}{% if infos.n_versions < infos.n_packaged|add:infos.n_overlay|mul:2 %}
@@ -22,3 +24,7 @@
 {% endif %}{% endif %}
 {{ infos.n_versions|sub:infos.n_packaged|sub:infos.n_overlay }}
 </td>
+
+<td>
+  {{ infos.n_packaged|add:infos.n_overlay|div:infos.n_versions|mul:100|floatformat:"0" }}
+</td>

euscanwww/templates/euscan/_packages.html

@@ -1,11 +1,18 @@
 {% load packages %}
+{% load sub %}
+{% load div %}
+{% load mul %}
 
 <table id="table" class="display">
   <thead>
     <th>Package</th>
     <th>Gentoo</th>
+    <th>Overlay</th>
+    <th>Upstream</th>
+    <th>Gentoo</th>
     <th>Overlays</th>
     <th>Unpackaged</th>
+    <th>Freshness</th>
   </thead>
   <tbody>
   {% for package in packages %}
@@ -16,6 +23,9 @@
       </a>
       {% package_bar package %}
     </td>
+    <td>{{ package.last_version_gentoo.version }}</td>
+    <td>{{ package.last_version_overlay.version }}</td>
+    <td>{{ package.last_version_upstream.version }}</td>
     {% package_cols package %}
   </tr>
   {% endfor %}

euscanwww/templates/euscan/categories.html

@@ -15,6 +15,7 @@
     <th>Gentoo</th>
     <th>Overlays</th>
     <th>Unpackaged</th>
+    <th>Freshness</th>
     {% if request.GET.extras %}
     <th>Graphs</th>
     {% endif %}

euscanwww/templates/euscan/herds.html

@@ -14,6 +14,7 @@
     <th>Gentoo</th>
     <th>Overlays</th>
     <th>Unpackaged</th>
+    <th>Freshness</th>
     {% if request.GET.extras %}
     <th>Graphs</th>
     {% endif %}

euscanwww/templates/euscan/maintainers.html

@@ -14,6 +14,7 @@
     <th>Gentoo</th>
     <th>Overlays</th>
     <th>Unpackaged</th>
+    <th>Freshness</th>
     {% if request.GET.extras %}
     <th>Graphs</th>
     {% endif %}