euscanwww: try to keep trace of versions change

Signed-off-by: Corentin Chary <corentincj@iksaif.net>

euscanwww/euscan/feeds.py

@@ -0,0 +1,146 @@
+from django.contrib.syndication.views import Feed, FeedDoesNotExist
+from django.shortcuts import get_object_or_404
+from django.http import Http404
+from django.utils.feedgenerator import Atom1Feed
+from django.core.urlresolvers import reverse
+
+from euscan.models import Version, Package, Herd, Maintainer, VersionLog
+from euscan.views import *
+
+class BaseFeed(Feed):
+    feed_type = Atom1Feed
+    author = 'euscan'
+
+    def item_title(self, vlog):
+        return str(vlog)
+
+    def item_description(self, vlog):
+        if vlog.overlay:
+            txt = 'Version %s-%s [%s] of package %s ' % (vlog.version, vlog.revision,
+                                                         vlog.slot, vlog.package)
+        else:
+            txt = 'Version %s of package %s ' % (vlog.version, vlog.package)
+        if vlog.action == vlog.VERSION_REMOVED:
+            if vlog.overlay:
+                txt += 'has been removed upstream'
+            else:
+                txt += 'has been removed from overlay "%s"' % vlog.overlay
+        if vlog.action == vlog.VERSION_ADDED:
+            if not vlog.overlay:
+                txt += 'has been added upstream'
+            else:
+                txt += 'has been added to overlay "%s"' % vlog.overlay
+
+        return txt
+
+    def item_link(self, vlog):
+        kwargs = {'category' : vlog.package.category, 'package' : vlog.package.name }
+        return reverse('euscan.views.package', kwargs=kwargs) + '#' + vlog.tag()
+
+    def item_pubdate(self, vlog):
+        return vlog.datetime
+
+    def item_categories(self, vlog):
+        return [vlog.package.category]
+
+class GlobalFeed(BaseFeed):
+    title = "euscan"
+    link = "/"
+    description = "Last euscan changes"
+
+    def categories(self):
+        categories = Package.objects.values('category').distinct();
+        return [ category['category'] for category in categories ]
+
+    def items(self):
+        return VersionLog.objects.order_by('-id')[:30]
+
+
+class PackageFeed(BaseFeed):
+    feed_type = Atom1Feed
+
+    def get_object(self, request, category, package):
+        return get_object_or_404(Package, category=category, name=package)
+
+    def title(self, package):
+        return "%s" % package
+
+    def description(self, package):
+        return "Last changes for %s" % package
+
+    def link(self, package):
+        return reverse('euscan.views.package', args=[package.category, package.name])
+
+    def description(self, package):
+        return package.description
+
+    def items(self, package):
+        return VersionLog.objects.filter(package=package).order_by('-id')[:30]
+
+    def item_description(self, vlog):
+        return ''
+
+class MaintainerFeed(BaseFeed):
+    feed_type = Atom1Feed
+
+    def get_object(self, request, maintainer_id):
+        return get_object_or_404(Maintainer, id=maintainer_id)
+
+    def title(self, maintainer):
+        return "%s" % maintainer
+
+    def description(self, maintainer):
+        return "Last changes for %s" % maintainer
+
+    def link(self, maintainer):
+        return reverse('euscan.views.maintainer', kwargs={'maintainer_id' : maintainer.id})
+
+    def description(self, maintainer):
+        return str(maintainer)
+
+    def items(self, maintainer):
+        q = VersionLog.objects.filter(package__maintainers__id=maintainer.id)
+        return q.order_by('-id')[:30]
+
+class HerdFeed(BaseFeed):
+    feed_type = Atom1Feed
+
+    def get_object(self, request, herd):
+        return get_object_or_404(Herd, herd=herd)
+
+    def title(self, herd):
+        return "%s" % herd
+
+    def description(self, herd):
+        return "Last changes for %s" % herd
+
+    def link(self, herd):
+        return reverse('euscan.views.herd', kwargs={'herd' : herd.herd})
+
+    def description(self, herd):
+        return str(herd)
+
+    def items(self, herd):
+        q = VersionLog.objects.filter(package__herds__id=herd.id)
+        return q.order_by('-id')[:30]
+
+class CategoryFeed(BaseFeed):
+    feed_type = Atom1Feed
+
+    def get_object(self, request, category):
+        if not Package.objects.filter(category=category).count():
+            raise FeedDoesNotExist
+        return category
+
+    def title(self, category):
+        return "%s" % category
+
+    def description(self, category):
+        return "Last changes for %s" % category
+
+    def link(self, category):
+        return reverse('euscan.views.category', args=[category])
+
+    def items(self, category):
+        q = VersionLog.objects.filter(package__category=category)
+        return q.order_by('-id')[:30]

euscanwww/euscan/management/commands/scan-metadata.py

@@ -14,6 +14,8 @@ from euscanwww.euscan.models import Package, Herd, Maintainer
 from gentoolkit.query import Query
 from gentoolkit.errors import GentoolkitFatalError
 
+from progressbar import ProgressBar, Bar, ETA, Percentage
+
 class Command(BaseCommand):
     _overlays = {}
 
@@ -28,6 +30,11 @@ class Command(BaseCommand):
             dest='quiet',
             default=False,
             help='Be quiet'),
+        make_option('--progress',
+            action='store_true',
+            dest='progress',
+            default=False,
+            help='Display progress'),
         )
     args = '<package package ...>'
     help = 'Scans metadata and fills database'
@@ -36,24 +43,41 @@ class Command(BaseCommand):
         if len(args) == 0 and options['all'] == False:
             raise CommandError('You must specify a package or use --all')
 
-        if not options['quiet']:
-            self.stdout.write('Scanning metadata...\n')
+        if not self.stdout.isatty():
+            options['progress'] = False
+
+        if options['progress']:
+            widgets = ['Scanning metadata: ', Percentage(), ' ', Bar(), ' ', ETA()]
+            if len(args):
+                count = len(args)
+            else:
+                count = Package.objects.count()
+            pbar = ProgressBar(widgets=widgets, maxval=count).start()
+            i = 0
+        else:
+            pbar = None
 
         if len(args) == 0:
             for pkg in Package.objects.all():
                 self.scan(options, '%s/%s' % (pkg.category, pkg.name))
+                if pbar:
+                    pbar.update(i)
+                    i += 1
         else:
             for package in args:
                 self.scan(options, package)
+                if pbar:
+                    pbar.update(i)
+                    i += 1
 
-        if not options['quiet']:
-            self.stdout.write('Done.\n')
+        if pbar:
+            pbar.finish()
 
     @commit_on_success
     def scan(self, options, query=None):
         matches = Query(query).find(
                 include_masked=True,
-                in_installed=False
+                in_installed=False,
         )
 
         if not matches:
@@ -73,6 +97,8 @@ class Command(BaseCommand):
         except GentoolkitFatalError, err:
             sys.stderr.write(self.style.ERROR("Gentoolkit fatal error: '%s'\n" % str(err)))
 
+        if created and not options['quiet']:
+            sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
         if pkg.metadata:
             obj.herds.clear()
             obj.maintainers.clear()
@@ -85,9 +111,6 @@ class Command(BaseCommand):
                 maintainer = self.store_maintainer(options, maintainer.name, maintainer.email)
                 obj.maintainers.add(maintainer)
 
-        if not options['quiet']:
-            sys.stdout.write('[p] %s/%s\n' % (pkg.category, pkg.name))
-
         obj.save()
 
     def store_herd(self, options, name, email):
@@ -99,7 +122,7 @@ class Command(BaseCommand):
 
         if created or herd.email != email:
             if not options['quiet']:
-                sys.stdout.write('[h] %s <%s>\n' % (name, email))
+                sys.stdout.write('+ [h] %s <%s>\n' % (name, email))
 
             herd.email = email
             herd.save()
@@ -116,7 +139,7 @@ class Command(BaseCommand):
 
         if created:
             if not options['quiet']:
-                sys.stdout.write('[m] %s <%s>\n' % (name.encode('utf-8'), email))
+                sys.stdout.write('+ [m] %s <%s>\n' % (name.encode('utf-8'), email))
 
             maintainer.name = name
             maintainer.save()

euscanwww/euscan/management/commands/scan-portage.py

@@ -9,7 +9,7 @@ from optparse import make_option
 
 from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand, CommandError
-from euscanwww.euscan.models import Package, Version
+from euscanwww.euscan.models import Package, Version, VersionLog
 
 class Command(BaseCommand):
     _overlays = {}
@@ -20,11 +20,16 @@ class Command(BaseCommand):
             dest='all',
             default=False,
             help='Scan all packages'),
-        make_option('--purge',
+        make_option('--purge-packages',
             action='store_true',
-            dest='purge',
+            dest='purge-packages',
             default=False,
             help='Purge old packages'),
+        make_option('--purge-versions',
+            action='store_true',
+            dest='purge-versions',
+            default=False,
+            help='Purge old versions'),
         make_option('--quiet',
             action='store_true',
             dest='quiet',
@@ -47,6 +52,9 @@ class Command(BaseCommand):
             for package in args:
                 self.scan(options, package)
 
+        if options['purge-versions']:
+            self.purge_versions(options)
+
         if not options['quiet']:
             self.stdout.write('Done.\n')
 
@@ -90,9 +98,9 @@ class Command(BaseCommand):
         if len(output) == 0:
             if not query:
                 return
-            if options['purge']:
+            if options['purge-packages']:
                 if not options['quiet']:
-                    sys.stdout.write('[gc] %s\n' % (query))
+                    sys.stdout.write('- [p] %s\n' % (query))
                 if '/' in query:
                     cat, pkg = portage.catsplit(query)
                     Package.objects.filter(category=cat, name=pkg).delete()
@@ -128,12 +136,12 @@ class Command(BaseCommand):
 
             self.store_version(options, package, cpv, slot, overlay)
 
-        if options['purge'] and not query:
+        if options['purge-packages'] and not query:
             for package in Package.objects.all():
                 cp = "%s/%s" % (package.category, package.name)
                 if cp not in packages:
                     if not options['quiet']:
-                        sys.stdout.write('[gc] %s\n' % (cp))
+                        sys.stdout.write('- [p] %s\n' % (package))
                     package.delete()
 
     def store_package(self, options, cat, pkg):
@@ -141,10 +149,10 @@ class Command(BaseCommand):
 
         if created:
             if not options['quiet']:
-                sys.stdout.write('[p] %s/%s\n' % (cat, pkg))
+                sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
 
-        # Delete previous versions to handle incremental scan correctly
-        Version.objects.filter(package=obj, packaged=True).delete()
+        ' Set all versions dead, then set found versions alive and delete old versions '
+        Version.objects.filter(package=obj, packaged=True).update(alive=False)
 
         obj.n_packaged = 0
         obj.n_overlay = 0
@@ -163,23 +171,53 @@ class Command(BaseCommand):
         else:
             overlay = 'gentoo'
 
-        if not options['quiet']:
-            sys.stdout.write('[v] %s:%s [%s]\n' % (cpv, slot, overlay))
-
         obj, created = Version.objects.get_or_create(package=package, slot=slot,
                                                      revision=rev, version=ver,
                                                      overlay=overlay)
-
-        if created or not package.n_packaged:
-            if overlay == 'gentoo':
-                package.n_packaged += 1
-            else:
-                package.n_overlay += 1
-        if created:
-            package.n_versions += 1
-
-        package.save()
-
+        obj.alive = True
         obj.packaged = True
         obj.save()
 
+        ''' nothing to do (note: it can't be an upstream version because overlay can't be empty here) '''
+        if not created:
+            return
+
+        if not options['quiet']:
+            sys.stdout.write('+ [v] %s \n' % (obj))
+
+        if overlay == 'gentoo':
+            package.n_packaged += 1
+        else:
+            package.n_overlay += 1
+        package.n_versions += 1
+        package.save()
+
+        entry = VersionLog.objects.create(package=obj.package, action=VersionLog.VERSION_ADDED)
+        entry.slot = obj.slot
+        entry.revision = obj.revision
+        entry.version = obj.version
+        entry.overlay = obj.overlay
+        entry.save()
+
+    @commit_on_success
+    def purge_versions(self, options):
+        ' For each dead versions '
+        for version in Version.objects.filter(packaged=True, alive=False):
+            entry = VersionLog.objects.create(package=version.package, action=VersionLog.VERSION_REMOVED)
+            entry.slot = version.slot
+            entry.revision = version.revision
+            entry.version = version.version
+            entry.overlay = version.overlay
+            entry.save()
+
+            if version.overlay == 'gentoo':
+                version.package.n_packaged -= 1
+            else:
+                version.package.n_overlay -= 1
+            version.package.n_versions -= 1
+            version.package.save()
+
+            if not options['quiet']:
+                sys.stdout.write('- [v] %s\n' % (version))
+        Version.objects.filter(packaged=False, alive=False).delete()
+

euscanwww/euscan/management/commands/scan-upstream.py

@@ -11,7 +11,7 @@ from optparse import make_option
 
 from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand, CommandError
-from euscanwww.euscan.models import Package, Version, EuscanResult
+from euscanwww.euscan.models import Package, Version, EuscanResult, VersionLog
 
 class Command(BaseCommand):
     _overlays = {}
@@ -27,6 +27,11 @@ class Command(BaseCommand):
             dest='feed',
             default=False,
             help='Read euscan output from stdin'),
+        make_option('--purge-versions',
+            action='store_true',
+            dest='purge-versions',
+            default=False,
+            help='Purge old versions'),
         make_option('--quiet',
             action='store_true',
             dest='quiet',
@@ -42,6 +47,8 @@ class Command(BaseCommand):
 
         if options['feed']:
             self.parse_output(options, sys.stdin)
+            if options['purge-versions']:
+                self.purge_versions(options)
             return
 
         if not options['quiet']:
@@ -57,10 +64,12 @@ class Command(BaseCommand):
 
         self.scan(options, packages)
 
+        if options['purge-versions']:
+            self.purge_versions(options)
+
         if not options['quiet']:
             self.stdout.write('Done.\n')
 
-    @commit_on_success
     def scan(self, options, packages=None):
         for package in packages:
             cmd = ['euscan', package]
@@ -70,6 +79,7 @@ class Command(BaseCommand):
 
             self.parse_output(options, output)
 
+    @commit_on_success
     def parse_output(self, options, output):
         from portage.versions import _cp
 
@@ -120,12 +130,11 @@ class Command(BaseCommand):
 
         obj, created = Package.objects.get_or_create(category=cat, name=pkg)
 
-        if created:
-            if not options['quiet']:
-                sys.stdout.write('[p] %s/%s\n' % (cat, pkg))
+        if created and not options['quiet']:
+            sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
 
-        # Delete previous versions to handle incremental scan correctly
-        Version.objects.filter(package=obj, packaged=False).delete()
+        ' Set all versions dead, then set found versions alive and delete old versions '
+        Version.objects.filter(package=obj, packaged=False).update(alive=False)
 
         obj.n_versions = Version.objects.filter(package=obj).count()
         obj.save()
@@ -137,15 +146,44 @@ class Command(BaseCommand):
                                                      revision='r0', version=ver,
                                                      overlay='')
 
-        if created or not obj.packaged:
+        obj.alive = True
+        obj.urls = url
+        obj.packaged = False
+        obj.save()
+
+        ''' If it's not a new version, just update the object and continue '''
+        if not created:
+            return
+
+        if not options['quiet']:
+            sys.stdout.write('+ [u] %s %s\n' % (obj, url))
+
+        entry = VersionLog.objects.create(package=package, action=VersionLog.VERSION_ADDED)
+        entry.slot = ''
+        entry.revision = 'r0'
+        entry.version = ver
+        entry.overlay = ''
+        entry.save()
+
+        package.n_versions += 1
+        package.save()
+
+
+    @commit_on_success
+    def purge_versions(self, options):
+        ' For each dead versions '
+        for version in Version.objects.filter(packaged=False, alive=False):
+            entry = VersionLog.objects.create(package=version.package, action=VersionLog.VERSION_REMOVED)
+            entry.slot = version.slot
+            entry.revision = version.revision
+            entry.version = version.version
+            entry.overlay = version.overlay
+            entry.save()
+
+            version.package.n_versions -= 1
+            version.package.save()
+
             if not options['quiet']:
-                sys.stdout.write('[u] %s/%s-%s %s\n' % (package.category, package.name,
-                                                        ver, url))
+                sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
+        Version.objects.filter(packaged=False, alive=False).delete()
 
-            obj.urls = url
-            obj.packaged = False
-            obj.save()
-
-        if created:
-            package.n_versions += 1
-            package.save()

euscanwww/euscan/management/commands/update-counters.py

@@ -29,8 +29,10 @@ class Command(BaseCommand):
         herds = {}
         maintainers = {}
 
-        # Could be done using raw SQL queries, but I don't have time for that
-        # right now ...
+        '''
+        Could be done using raw SQL queries, but I don't have time for that
+        right now ...
+        '''
 
         wlog = WorldLog()
         wlog.datetime = now
@@ -100,19 +102,19 @@ class Command(BaseCommand):
 
         for clog in categories.values():
             if not options['quiet']:
-                self.stdout.write('[c] %s\n' % clog)
+                self.stdout.write('+ [cl] %s\n' % clog)
             charts.rrd_update('category-%s' % clog.category, now, clog)
             clog.save()
 
         for hlog in herds.values():
             if not options['quiet']:
-                self.stdout.write('[h] %s\n' % hlog)
+                self.stdout.write('+ [hl] %s\n' % hlog)
             charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
             hlog.save()
 
         for mlog in maintainers.values():
             if not options['quiet']:
-                self.stdout.write('[m] %s\n' % mlog)
+                self.stdout.write('+ [ml] %s\n' % mlog)
             charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
             mlog.save()
 

euscanwww/euscan/migrations/0007_auto__add_versionlog__add_field_version_alive.py

@@ -0,0 +1,127 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding model 'VersionLog'
+        db.create_table('euscan_versionlog', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('package', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['euscan.Package'])),
+            ('datetime', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime(2011, 8, 22, 21, 41, 33, 285480))),
+            ('slot', self.gf('django.db.models.fields.CharField')(max_length=128)),
+            ('revision', self.gf('django.db.models.fields.CharField')(max_length=128)),
+            ('version', self.gf('django.db.models.fields.CharField')(max_length=128)),
+            ('packaged', self.gf('django.db.models.fields.BooleanField')(default=False)),
+            ('overlay', self.gf('django.db.models.fields.CharField')(default='gentoo', max_length=128)),
+            ('action', self.gf('django.db.models.fields.IntegerField')()),
+        ))
+        db.send_create_signal('euscan', ['VersionLog'])
+
+        # Adding field 'Version.alive'
+        db.add_column('euscan_version', 'alive', self.gf('django.db.models.fields.BooleanField')(default=True, db_index=True), keep_default=False)
+
+
+    def backwards(self, orm):
+        
+        # Deleting model 'VersionLog'
+        db.delete_table('euscan_versionlog')
+
+        # Deleting field 'Version.alive'
+        db.delete_column('euscan_version', 'alive')
+
+
+    models = {
+        'euscan.categorylog': {
+            'Meta': {'object_name': 'CategoryLog', '_ormbases': ['euscan.Log']},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'euscan.euscanresult': {
+            'Meta': {'object_name': 'EuscanResult'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
+            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'euscan.herd': {
+            'Meta': {'object_name': 'Herd'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
+            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'euscan.herdlog': {
+            'Meta': {'object_name': 'HerdLog', '_ormbases': ['euscan.Log']},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Herd']"}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'euscan.log': {
+            'Meta': {'object_name': 'Log'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'euscan.maintainer': {
+            'Meta': {'object_name': 'Maintainer'},
+            'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'euscan.maintainerlog': {
+            'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['euscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Maintainer']"})
+        },
+        'euscan.package': {
+            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
+            'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'euscan.version': {
+            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
+            'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'euscan.versionlog': {
+            'Meta': {'object_name': 'VersionLog'},
+            'action': ('django.db.models.fields.IntegerField', [], {}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2011, 8, 22, 21, 41, 33, 285480)'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'euscan.worldlog': {
+            'Meta': {'object_name': 'WorldLog', '_ormbases': ['euscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['euscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        }
+    }
+
+    complete_apps = ['euscan']

euscanwww/euscan/models.py

@@ -1,4 +1,5 @@
 from django.db import models
+from datetime import datetime
 
 class Herd(models.Model):
     herd = models.CharField(max_length=128, unique=True)
@@ -24,7 +25,7 @@ class Package(models.Model):
     herds = models.ManyToManyField(Herd, blank=True)
     maintainers = models.ManyToManyField(Maintainer, blank=True)
 
-    # For performance, we keep pre-computed counters
+    ' For performance, we keep pre-computed counters '
     n_versions = models.IntegerField(default=0)
     n_packaged = models.IntegerField(default=0)
     n_overlay = models.IntegerField(default=0)
@@ -43,6 +44,7 @@ class Version(models.Model):
     packaged = models.BooleanField()
     overlay = models.CharField(max_length=128, default='gentoo')
     urls = models.TextField(blank=True)
+    alive = models.BooleanField(default=True, db_index=True)
 
     def __unicode__(self):
         return '%s/%s-%s-%s:%s [%s]' % (self.package.category, self.package.name,
@@ -52,6 +54,34 @@ class Version(models.Model):
     class Meta:
         unique_together = ['package', 'slot', 'revision', 'version', 'overlay']
 
+class VersionLog(models.Model):
+    VERSION_ADDED = 1
+    VERSION_REMOVED = 2
+    VERSION_ACTIONS = (
+        (VERSION_ADDED, 'Added'),
+        (VERSION_REMOVED, 'Removed')
+    )
+
+    package = models.ForeignKey(Package)
+    datetime = models.DateTimeField(default=datetime.now())
+    slot = models.CharField(max_length=128)
+    revision = models.CharField(max_length=128)
+    version = models.CharField(max_length=128)
+    packaged = models.BooleanField()
+    overlay = models.CharField(max_length=128, default='gentoo')
+    action = models.IntegerField(choices=VERSION_ACTIONS)
+
+    def tag(self):
+        return '%s-%s:%s-[%s]' % (self.version, self.revision, self.slot,
+                                  self.overlay)
+
+    def __unicode__(self):
+        txt = '+ ' if self.action == self.VERSION_ADDED else '- '
+        txt += '%s/%s-%s-%s:%s [%s]' % (self.package.category, self.package.name,
+                                        self.version, self.revision, self.slot,
+                                        self.overlay if self.overlay else '<upstream>')
+        return txt
+
 class EuscanResult(models.Model):
     package = models.ForeignKey(Package)
     datetime = models.DateTimeField()
@@ -61,13 +91,19 @@ class EuscanResult(models.Model):
 class Log(models.Model):
     datetime = models.DateTimeField()
 
-    n_packages_gentoo   = models.IntegerField(default=0) # Packages up to date in the main portage tree
-    n_packages_overlay  = models.IntegerField(default=0) # Packages up to date in an overlay
-    n_packages_outdated = models.IntegerField(default=0) # Packages outdated
+    ' Packages up to date in the main portage tree '
+    n_packages_gentoo   = models.IntegerField(default=0)
+    ' Packages up to date in an overlay '
+    n_packages_overlay  = models.IntegerField(default=0)
+    ' Packages outdated '
+    n_packages_outdated = models.IntegerField(default=0)
 
-    n_versions_gentoo   = models.IntegerField(default=0) # Versions in the main portage tree
-    n_versions_overlay  = models.IntegerField(default=0) # Versions in overlays
-    n_versions_upstream = models.IntegerField(default=0) # Upstream versions, not in the main tree or overlays
+    ' Versions in the main portage tree '
+    n_versions_gentoo   = models.IntegerField(default=0)
+    ' Versions in overlays '
+    n_versions_overlay  = models.IntegerField(default=0)
+    ' Upstream versions, not in the main tree or overlays '
+    n_versions_upstream = models.IntegerField(default=0)
 
     def __unicode__(self):
         return u'[%d:%d:%d] [%d:%d:%d]' % \

euscanwww/euscan/urls.py

@@ -1,20 +1,42 @@
 from django.conf.urls.defaults import *
+from feeds import *
+
+package_patterns = patterns('euscan.views',
+    url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/feed/$', PackageFeed(), name='package_feed'),
+    (r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/$', 'package'),
+)
+
+categories_patterns = patterns('euscan.views',
+    (r'(?P<category>[\w+][\w+.-]*)/view/$', 'category'),
+    url(r'(?P<category>[\w+][\w+.-]*)/feed/$', CategoryFeed(), name='category_feed'),
+    (r'(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', 'chart_category'),
+    (r'$', 'categories'),
+)
+
+herds_patterns = patterns('euscan.views',
+    (r'(?P<herd>[\@\{\}\w+.-]*)/view/$', 'herd'),
+    url(r'(?P<herd>[\@\{\}\w+.-]*)/feed/$', HerdFeed(), name='herd_feed'),
+    (r'(?P<herd>[\@\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', 'chart_herd'),
+    (r'$', 'herds'),
+)
+
+maintainers_patterns = patterns('euscan.views',
+    (r'(?P<maintainer_id>\d+)/view/$', 'maintainer'),
+    url(r'(?P<maintainer_id>\d+)/feed/$', MaintainerFeed(), name='maintainer_feed'),
+    (r'(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$', 'chart_maintainer'),
+    (r'$', 'maintainers'),
+)
 
 urlpatterns = patterns('euscan.views',
     (r'^$', 'index'),
+    url(r'^feed/$', GlobalFeed(), name='global_feed'),
     (r'^about/$', 'about'),
     (r'^statistics/$', 'statistics'),
     (r'^statistics/charts/(?P<chart>[\w\-]+).png$', 'chart'),
     (r'^world/$', 'world'),
     (r'^world/scan/$', 'world_scan'),
-    (r'^categories/$', 'categories'),
-    (r'^categories/(?P<category>[\w+][\w+.-]*)/view/$', 'category'),
-    (r'^categories/(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', 'chart_category'),
-    (r'^herds/$', 'herds'),
-    (r'^herds/(?P<herd>[\{\}\w+.-]*)/view/$', 'herd'),
-    (r'^herds/(?P<herd>[\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', 'chart_herd'),
-    (r'^maintainers/$', 'maintainers'),
-    (r'^maintainers/(?P<maintainer_id>\d+)/view/$', 'maintainer'),
-    (r'^maintainers/(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$', 'chart_maintainer'),
-    (r'^package/(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/$', 'package'),
+    (r'^categories/', include(categories_patterns)),
+    (r'^herds/', include(herds_patterns)),
+    (r'^maintainers/', include(maintainers_patterns)),
+    (r'^package/', include(package_patterns)),
 )

euscanwww/euscan/views.py

@@ -1,10 +1,9 @@
 from annoying.decorators import render_to
-from django.http import HttpResponse
-from django.http import Http404
+from django.http import HttpResponse, Http404
 from django.shortcuts import get_object_or_404
 from django.db.models import Sum, Max
 
-from euscan.models import Version, Package, Herd, Maintainer, EuscanResult
+from euscan.models import Version, Package, Herd, Maintainer, EuscanResult, VersionLog
 from euscan.forms import WorldForm, PackagesForm
 
 import charts
@@ -82,8 +81,9 @@ def package(request, category, package):
     upstream = Version.objects.filter(package=package, packaged=False)
     log = EuscanResult.objects.filter(package=package).order_by('-datetime')[:1]
     log = log[0] if log else None
+    vlog = VersionLog.objects.filter(package=package).order_by('-id')
     return { 'package' : package, 'packaged' : packaged,
-             'upstream' : upstream, 'log' : log }
+             'upstream' : upstream, 'log' : log, 'vlog' : vlog }
 
 @render_to('euscan/world.html')
 def world(request):

euscanwww/media/css/style.css

@@ -216,6 +216,14 @@ th
   border: 1px dotted #5682AD;
   }
 
+.added {
+  color: #262;
+}
+
+.removed {
+  color: #F00;
+}
+
 .err
 {
   border-color: #F00;
@@ -259,3 +267,8 @@ th
 .package_stat .upstream {
     background: #FDEADD;
 }
+
+.log {
+    max-height: 100pt;
+    overflow: auto;
+}

euscanwww/templates/_base.html

@@ -3,6 +3,9 @@
   <head>
     <title>{% block title %}euscan{% endblock %}</title>
     <meta http-equiv="content-type" content="text/html; charset=utf-8" />
+    {% block meta %}
+    <link rel="alternate" type="application/atom+xml" title="Global log" href="{% url global_feed %}" />
+    {% endblock %}
     {% block css %}
     <link rel="stylesheet" type="text/css" href="{{ MEDIA_URL }}css/style.css" media="screen" title="Normal" />
     {% endblock %}
@@ -36,6 +39,13 @@
 	  <li><a href="#">Register</a></li>
 	  -->
 	  <li>---</li>
+	  {% block menu_feed %}
+	  <li>
+	    <img src="{{ MEDIA_URL }}/img/feed.png" alt="feed" />
+	    <a href="Global Feed" href="{% url global_feed %}">Global Feed</a>
+	  </li>
+	  {% endblock %}
+	  <li>---</li>
 	  <li><a href="{% url euscan.views.about %}">About</a></li>
 	  {% endblock %}
 	</ul>

euscanwww/templates/euscan/category.html

@@ -6,6 +6,14 @@
 {{ block.super }} - Category: {{ category }}
 {% endblock %}
 
+{% block menu_feed %}
+{{ block.super }}
+<li>
+  <img src="{{ MEDIA_URL }}/img/feed.png" alt="feed" />
+  <a title="{{ category }} Feed" href="{% url category_feed category %}">{{ category }}</a>
+</li>
+{% endblock %}
+
 {% block content %}
 <h2>Category: {{ category }}</h2>
 {% packages packages %}

euscanwww/templates/euscan/herd.html

@@ -6,6 +6,14 @@
 {{ block.super }} - Herd: {{ herd.herd }}
 {% endblock %}
 
+{% block menu_feed %}
+{{ block.super }}
+<li>
+  <img src="{{ MEDIA_URL }}/img/feed.png" alt="feed" />
+  <a title="{{ herd.herd }} Feed" href="{% url herd_feed herd.herd %}">{{ herd.herd }}</a>
+</li>
+{% endblock %}
+
 {% block content %}
 <h2>Herd: {{ herd.herd }}</h2>
 {% packages packages %}

euscanwww/templates/euscan/maintainer.html

@@ -6,6 +6,14 @@
 {{ block.super }} - Maintainer: {{ maintainer.name }}
 {% endblock %}
 
+{% block menu_feed %}
+{{ block.super }}
+<li>
+  <img src="{{ MEDIA_URL }}/img/feed.png" alt="feed" />
+  <a title="{{ maintainer.name }} Feed" href="{% url maintainer_feed maintainer.id %}">{{ maintainer.name }}</a>
+</li>
+{% endblock %}
+
 {% block content %}
 <h2>Maintainer: {{ maintainer.name }} &lt{{ maintainer.email }}&gt</h2>
 {% packages packages %}

euscanwww/templates/euscan/package.html

@@ -2,6 +2,19 @@
 
 {% load sub %}
 
+{% block meta %}
+{{ block.super }}
+<link rel="alternate" type="application/atom+xml" title="{{ package }} Feed" href="{% url package_feed package.category package.name %}" />
+{% endblock %}
+
+{% block menu_feed %}
+{{ block.super }}
+<li>
+  <img src="{{ MEDIA_URL }}/img/feed.png" alt="feed" />
+  <a title="{{ package }} Feed" href="{% url package_feed package.category package.name %}">{{ package }}</a>
+</li>
+{% endblock %}
+
 {% block title %}
 {{ block.super }} - {{ package.category }}/{{ package.name }}
 {% endblock %}
@@ -73,11 +86,25 @@
      </ul>
   </dd>
   {% endif %}
+  <dt>Version history</dt>
+  <dd>
+    <ul class="log">
+      {% for version in vlog %}
+      {% if version.action == version.VERSION_ADDED %}
+      <li class="added">
+      {% else %}
+      <li class="removed">
+      {% endif %}
+	{{ version }} - {{ version.datetime }}
+      </li>
+      {% endfor %}
+    </ul>
+  </dt>
   {% if log %}
   <dt>euscan log</dt>
   <dd>
     <p>Date: {{ log.datetime }}
-    <pre>
+    <pre class="log">
       {{ log.result }}
     </pre>
   </dd>