euscanwww: implement scan-upstream, fix other commands

Also break migrations :).

Signed-off-by: Corentin Chary <corentincj@iksaif.net>

euscanwww/euscan/management/commands/scan-metadata.py

@@ -12,7 +12,7 @@ from django.core.management.base import BaseCommand, CommandError
 from euscanwww.euscan.models import Package, Herd, Maintainer
 
 from gentoolkit.query import Query
-from gentoolkit.eclean.search import (port_settings)
+from gentoolkit.errors import GentoolkitFatalError
 
 class Command(BaseCommand):
     _overlays = {}
@@ -37,7 +37,7 @@ class Command(BaseCommand):
             raise CommandError('You must specify a package or use --all')
 
         if not options['quiet']:
-            self.stdout.write('Scanning portage tree...\n')
+            self.stdout.write('Scanning metadata...\n')
 
         if len(args) == 0:
             for pkg in Package.objects.all():
@@ -62,21 +62,25 @@ class Command(BaseCommand):
 
 	matches = sorted(matches)
         pkg = matches.pop()
-	if pkg.version == '9999':
+	if pkg.version == '9999' and len(matches):
 		pkg = matches.pop()
 
         obj, created = Package.objects.get_or_create(category=pkg.category, name=pkg.name)
 
-        obj.homepage = pkg.environment("HOMEPAGE")
+        try:
-        obj.description = pkg.environment("DESCRIPTION")
+            obj.homepage = pkg.environment("HOMEPAGE")
+            obj.description = pkg.environment("DESCRIPTION")
+        except GentoolkitFatalError, err:
+            sys.stderr.write(self.style.ERROR("Gentoolkit fatal error: '%s'\n" % str(err)))
 
-        for herd in pkg.metadata.herds(True):
+        if pkg.metadata:
-            herd = self.store_herd(options, herd[0], herd[1])
+            for herd in pkg.metadata.herds(True):
-            obj.herds.add(herd)
+                herd = self.store_herd(options, herd[0], herd[1])
+                obj.herds.add(herd)
 
-        for maintainer in pkg.metadata.maintainers():
+            for maintainer in pkg.metadata.maintainers():
-            maintainer = self.store_maintainer(options, maintainer.name, maintainer.email)
+                maintainer = self.store_maintainer(options, maintainer.name, maintainer.email)
-            obj.maintainers.add(maintainer)
+                obj.maintainers.add(maintainer)
 
         if not options['quiet']:
             sys.stdout.write('[p] %s/%s\n' % (pkg.category, pkg.name))

euscanwww/euscan/management/commands/scan-portage.py

@@ -36,9 +36,6 @@ class Command(BaseCommand):
         if not options['quiet']:
             self.stdout.write('Scanning portage tree...\n')
 
-        Version.objects.all().delete()
-        Package.objects.all().delete()
-
         for package in args:
             self.scan(options, package)
 
@@ -119,11 +116,13 @@ class Command(BaseCommand):
         if created:
             if not options['quiet']:
                 sys.stdout.write('[p] %s/%s\n' % (cat, pkg))
-            obj.save()
 
         # Delete previous versions to handle incremental scan correctly
         Version.objects.filter(package=obj, packaged=True).delete()
 
+        obj.n_versions = Version.objects.filter(package=obj).count()
+        obj.save()
+
         return obj
 
     def store_version(self, options, package, cpv, slot, overlay):
@@ -143,5 +142,8 @@ class Command(BaseCommand):
                                                      revision=rev, version=ver,
                                                      overlay=overlay)
         obj.packaged = True
-        #obj.save()
+        obj.save()
 
+        package.n_versions += 1
+        package.n_packaged += 1
+        package.save()

euscanwww/euscan/management/commands/scan-upstream.py

@@ -0,0 +1,147 @@
+import subprocess
+import portage
+import sys
+import os
+import re
+
+from datetime import datetime
+from portage import versions
+from optparse import make_option
+
+from django.db.transaction import commit_on_success
+from django.core.management.base import BaseCommand, CommandError
+from euscanwww.euscan.models import Package, Version, EuscanResult
+
+class Command(BaseCommand):
+    _overlays = {}
+
+    option_list = BaseCommand.option_list + (
+        make_option('--all',
+            action='store_true',
+            dest='all',
+            default=False,
+            help='Scan all packages'),
+        make_option('--parallel',
+            action='store_true',
+            dest='parallel',
+            default=False,
+            help='Use GNU Parallel'),
+        make_option('--quiet',
+            action='store_true',
+            dest='quiet',
+            default=False,
+            help='Be quiet'),
+        )
+    args = '<package package ...>'
+    help = 'Scans metadata and fills database'
+
+    def handle(self, *args, **options):
+        if len(args) == 0 and options['all'] == False:
+            raise CommandError('You must specify a package or use --all')
+
+        if not options['quiet']:
+            self.stdout.write('Scanning upstream...\n')
+
+        packages = []
+
+        if len(args) == 0:
+            for pkg in Package.objects.all():
+                packages.append('%s/%s' % (pkg.category, pkg.name))
+        else:
+            packages = list(args)
+
+        self.scan(options, packages)
+
+        if not options['quiet']:
+            self.stdout.write('Done.\n')
+
+    @commit_on_success
+    def scan(self, options, packages=None):
+        if options['parallel']:
+            jobs = '\n'.join(packages)
+            cmd = ['gparallel', '--jobs', '150%', 'euscan']
+
+            fp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+            output = fp.communicate(jobs)[0]
+
+            self.parse_output(options, output)
+        else:
+            for package in packages:
+                cmd = ['euscan', package]
+
+                fp = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+                output = fp.communicate()[0]
+
+                self.parse_output(options, output)
+
+    def parse_output(self, options, output):
+        from portage.versions import _cp
+
+        package_re = re.compile(r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$')
+        version_re = re.compile(r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$')
+
+        package = None
+        log = ""
+
+        for line in output.split('\n'):
+            match = package_re.match(line)
+            if match:
+                if package:
+                    self.store_result(options, package, log)
+
+                cpv = match.group('cpv')
+                package = self.store_package(options, cpv)
+                log = line
+                continue
+
+            log += line
+
+            match = version_re.match(line)
+            if match:
+                ver = match.group('ver')
+                url = match.group('url')
+                self.store_version(options, package, ver, url)
+
+        if package:
+            self.store_result(options, package, log)
+
+    def store_result(self, options, package, log):
+        obj = EuscanResult()
+        obj.package = package
+        obj.result = log
+        obj.datetime = datetime.now()
+        obj.save()
+
+    def store_package(self, options, cpv):
+        cat, pkg, ver, rev = portage.catpkgsplit(cpv)
+
+        obj, created = Package.objects.get_or_create(category=cat, name=pkg)
+
+        if created:
+            if not options['quiet']:
+                sys.stdout.write('[p] %s/%s\n' % (cat, pkg))
+
+        # Delete previous versions to handle incremental scan correctly
+        Version.objects.filter(package=obj, packaged=False).delete()
+
+        obj.n_versions = Version.objects.filter(package=obj).count()
+        obj.save()
+
+        return obj
+
+    def store_version(self, options, package, ver, url):
+        obj, created = Version.objects.get_or_create(package=package, slot='',
+                                                     revision='r0', version=ver,
+                                                     overlay='')
+
+        if created or not obj.packaged:
+            if not options['quiet']:
+                sys.stdout.write('[u] %s/%s-%s %s\n' % (package.category, package.name,
+                                                        ver, url))
+
+            obj.url = url
+            obj.packaged = False
+            obj.save()
+
+        package.n_versions += 1
+        package.save()

euscanwww/euscan/migrations/0001_initial.py

@@ -12,6 +12,7 @@ class Migration(SchemaMigration):
         db.create_table('euscan_herd', (
             ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
             ('herd', self.gf('django.db.models.fields.CharField')(unique=True, max_length=128)),
+            ('email', self.gf('django.db.models.fields.CharField')(max_length=128, null=True, blank=True)),
         ))
         db.send_create_signal('euscan', ['Herd'])
 
@@ -33,6 +34,8 @@ class Migration(SchemaMigration):
             ('name', self.gf('django.db.models.fields.CharField')(max_length=128)),
             ('description', self.gf('django.db.models.fields.TextField')(blank=True)),
             ('homepage', self.gf('django.db.models.fields.CharField')(max_length=256, blank=True)),
+            ('n_versions', self.gf('django.db.models.fields.IntegerField')(default=0)),
+            ('n_packaged', self.gf('django.db.models.fields.IntegerField')(default=0)),
         ))
         db.send_create_signal('euscan', ['Package'])
 
@@ -68,24 +71,56 @@ class Migration(SchemaMigration):
         ))
         db.send_create_signal('euscan', ['Version'])
 
-        # Adding unique constraint on 'Version', fields ['package', 'slot', 'revision', 'version']
+        # Adding unique constraint on 'Version', fields ['package', 'slot', 'revision', 'version', 'overlay']
-        db.create_unique('euscan_version', ['package_id', 'slot', 'revision', 'version'])
+        db.create_unique('euscan_version', ['package_id', 'slot', 'revision', 'version', 'overlay'])
 
         # Adding model 'EuscanResult'
         db.create_table('euscan_euscanresult', (
             ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
             ('package', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['euscan.Package'])),
-            ('startdate', self.gf('django.db.models.fields.DateTimeField')()),
+            ('datetime', self.gf('django.db.models.fields.DateTimeField')()),
-            ('endstate', self.gf('django.db.models.fields.DateTimeField')()),
             ('result', self.gf('django.db.models.fields.TextField')(blank=True)),
         ))
         db.send_create_signal('euscan', ['EuscanResult'])
 
+        # Adding model 'CategoryLog'
+        db.create_table('euscan_categorylog', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('category', self.gf('django.db.models.fields.CharField')(max_length=128)),
+            ('datetime', self.gf('django.db.models.fields.DateTimeField')()),
+            ('n_packages', self.gf('django.db.models.fields.IntegerField')(default=0)),
+            ('n_versions', self.gf('django.db.models.fields.IntegerField')(default=0)),
+            ('n_packaged', self.gf('django.db.models.fields.IntegerField')(default=0)),
+        ))
+        db.send_create_signal('euscan', ['CategoryLog'])
+
+        # Adding model 'HerdLog'
+        db.create_table('euscan_herdlog', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('herd', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['euscan.Herd'])),
+            ('datetime', self.gf('django.db.models.fields.DateTimeField')()),
+            ('n_packages', self.gf('django.db.models.fields.IntegerField')(default=0)),
+            ('n_versions', self.gf('django.db.models.fields.IntegerField')(default=0)),
+            ('n_packaged', self.gf('django.db.models.fields.IntegerField')(default=0)),
+        ))
+        db.send_create_signal('euscan', ['HerdLog'])
+
+        # Adding model 'MaintainerLog'
+        db.create_table('euscan_maintainerlog', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('maintainer', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['euscan.Maintainer'])),
+            ('datetime', self.gf('django.db.models.fields.DateTimeField')()),
+            ('n_packages', self.gf('django.db.models.fields.IntegerField')(default=0)),
+            ('n_versions', self.gf('django.db.models.fields.IntegerField')(default=0)),
+            ('n_packaged', self.gf('django.db.models.fields.IntegerField')(default=0)),
+        ))
+        db.send_create_signal('euscan', ['MaintainerLog'])
+
 
     def backwards(self, orm):
         
-        # Removing unique constraint on 'Version', fields ['package', 'slot', 'revision', 'version']
+        # Removing unique constraint on 'Version', fields ['package', 'slot', 'revision', 'version', 'overlay']
-        db.delete_unique('euscan_version', ['package_id', 'slot', 'revision', 'version'])
+        db.delete_unique('euscan_version', ['package_id', 'slot', 'revision', 'version', 'overlay'])
 
         # Removing unique constraint on 'Package', fields ['category', 'name']
         db.delete_unique('euscan_package', ['category', 'name'])
@@ -114,27 +149,63 @@ class Migration(SchemaMigration):
         # Deleting model 'EuscanResult'
         db.delete_table('euscan_euscanresult')
 
+        # Deleting model 'CategoryLog'
+        db.delete_table('euscan_categorylog')
+
+        # Deleting model 'HerdLog'
+        db.delete_table('euscan_herdlog')
+
+        # Deleting model 'MaintainerLog'
+        db.delete_table('euscan_maintainerlog')
+
 
     models = {
+        'euscan.categorylog': {
+            'Meta': {'object_name': 'CategoryLog'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
         'euscan.euscanresult': {
             'Meta': {'object_name': 'EuscanResult'},
-            'endstate': ('django.db.models.fields.DateTimeField', [], {}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
             'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
-            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
-            'startdate': ('django.db.models.fields.DateTimeField', [], {})
         },
         'euscan.herd': {
             'Meta': {'object_name': 'Herd'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
             'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
             'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
         },
+        'euscan.herdlog': {
+            'Meta': {'object_name': 'HerdLog'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Herd']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
         'euscan.maintainer': {
             'Meta': {'unique_together': "(['name', 'email'],)", 'object_name': 'Maintainer'},
             'email': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
             'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
         },
+        'euscan.maintainerlog': {
+            'Meta': {'object_name': 'MaintainerLog'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Maintainer']"}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
         'euscan.package': {
             'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
             'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
@@ -143,10 +214,12 @@ class Migration(SchemaMigration):
             'homepage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
             'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
             'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
         },
         'euscan.version': {
-            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version'],)", 'object_name': 'Version'},
+            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
             'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'overlay': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
             'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),

euscanwww/euscan/migrations/0002_auto__del_unique_version_slot_revision_version_package__add_unique_ver.py

@@ -1,70 +0,0 @@
-# encoding: utf-8
-import datetime
-from south.db import db
-from south.v2 import SchemaMigration
-from django.db import models
-
-class Migration(SchemaMigration):
-
-    def forwards(self, orm):
-        
-        # Removing unique constraint on 'Version', fields ['slot', 'revision', 'version', 'package']
-        db.delete_unique('euscan_version', ['slot', 'revision', 'version', 'package_id'])
-
-        # Adding unique constraint on 'Version', fields ['slot', 'overlay', 'revision', 'version', 'package']
-        db.create_unique('euscan_version', ['slot', 'overlay', 'revision', 'version', 'package_id'])
-
-
-    def backwards(self, orm):
-        
-        # Removing unique constraint on 'Version', fields ['slot', 'overlay', 'revision', 'version', 'package']
-        db.delete_unique('euscan_version', ['slot', 'overlay', 'revision', 'version', 'package_id'])
-
-        # Adding unique constraint on 'Version', fields ['slot', 'revision', 'version', 'package']
-        db.create_unique('euscan_version', ['slot', 'revision', 'version', 'package_id'])
-
-
-    models = {
-        'euscan.euscanresult': {
-            'Meta': {'object_name': 'EuscanResult'},
-            'endstate': ('django.db.models.fields.DateTimeField', [], {}),
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
-            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
-            'startdate': ('django.db.models.fields.DateTimeField', [], {})
-        },
-        'euscan.herd': {
-            'Meta': {'object_name': 'Herd'},
-            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
-        },
-        'euscan.maintainer': {
-            'Meta': {'unique_together': "(['name', 'email'],)", 'object_name': 'Maintainer'},
-            'email': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
-        },
-        'euscan.package': {
-            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
-            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
-            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
-            'homepage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
-        },
-        'euscan.version': {
-            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'overlay': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
-            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
-            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
-            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
-        }
-    }
-
-    complete_apps = ['euscan']

euscanwww/euscan/migrations/0003_auto__add_field_herd_email.py

@@ -1,65 +0,0 @@
-# encoding: utf-8
-import datetime
-from south.db import db
-from south.v2 import SchemaMigration
-from django.db import models
-
-class Migration(SchemaMigration):
-
-    def forwards(self, orm):
-        
-        # Adding field 'Herd.email'
-        db.add_column('euscan_herd', 'email', self.gf('django.db.models.fields.CharField')(default='', max_length=128), keep_default=False)
-
-
-    def backwards(self, orm):
-        
-        # Deleting field 'Herd.email'
-        db.delete_column('euscan_herd', 'email')
-
-
-    models = {
-        'euscan.euscanresult': {
-            'Meta': {'object_name': 'EuscanResult'},
-            'endstate': ('django.db.models.fields.DateTimeField', [], {}),
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
-            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
-            'startdate': ('django.db.models.fields.DateTimeField', [], {})
-        },
-        'euscan.herd': {
-            'Meta': {'object_name': 'Herd'},
-            'email': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
-        },
-        'euscan.maintainer': {
-            'Meta': {'unique_together': "(['name', 'email'],)", 'object_name': 'Maintainer'},
-            'email': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
-        },
-        'euscan.package': {
-            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
-            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
-            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
-            'homepage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
-        },
-        'euscan.version': {
-            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'overlay': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
-            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
-            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
-            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
-        }
-    }
-
-    complete_apps = ['euscan']

euscanwww/euscan/migrations/0004_auto__chg_field_herd_email.py

@@ -1,65 +0,0 @@
-# encoding: utf-8
-import datetime
-from south.db import db
-from south.v2 import SchemaMigration
-from django.db import models
-
-class Migration(SchemaMigration):
-
-    def forwards(self, orm):
-        
-        # Changing field 'Herd.email'
-        db.alter_column('euscan_herd', 'email', self.gf('django.db.models.fields.CharField')(max_length=128, null=True))
-
-
-    def backwards(self, orm):
-        
-        # Changing field 'Herd.email'
-        db.alter_column('euscan_herd', 'email', self.gf('django.db.models.fields.CharField')(default='', max_length=128))
-
-
-    models = {
-        'euscan.euscanresult': {
-            'Meta': {'object_name': 'EuscanResult'},
-            'endstate': ('django.db.models.fields.DateTimeField', [], {}),
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
-            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
-            'startdate': ('django.db.models.fields.DateTimeField', [], {})
-        },
-        'euscan.herd': {
-            'Meta': {'object_name': 'Herd'},
-            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
-            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
-        },
-        'euscan.maintainer': {
-            'Meta': {'unique_together': "(['name', 'email'],)", 'object_name': 'Maintainer'},
-            'email': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
-        },
-        'euscan.package': {
-            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
-            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
-            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
-            'homepage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['euscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
-        },
-        'euscan.version': {
-            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'overlay': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['euscan.Package']"}),
-            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
-            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
-            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
-        }
-    }
-
-    complete_apps = ['euscan']

euscanwww/euscan/models.py

@@ -27,6 +27,10 @@ class Package(models.Model):
     herds = models.ManyToManyField(Herd, blank=True)
     maintainers = models.ManyToManyField(Maintainer, blank=True)
 
+    # For performance, we keep pre-computed counters
+    n_versions = models.IntegerField(default=0)
+    n_packaged = models.IntegerField(default=0)
+
     def __unicode__(self):
         return '%s/%s' % (self.category, self.name)
 
@@ -52,6 +56,30 @@ class Version(models.Model):
 
 class EuscanResult(models.Model):
     package = models.ForeignKey(Package)
-    startdate = models.DateTimeField()
+    datetime = models.DateTimeField()
-    endstate = models.DateTimeField()
     result = models.TextField(blank=True)
+
+# Keep data for charts
+class CategoryLog(models.Model):
+    category = models.CharField(max_length=128)
+    datetime = models.DateTimeField()
+
+    n_packages = models.IntegerField(default=0)
+    n_versions = models.IntegerField(default=0)
+    n_packaged = models.IntegerField(default=0)
+
+class HerdLog(models.Model):
+    herd = models.ForeignKey(Herd)
+    datetime = models.DateTimeField()
+
+    n_packages = models.IntegerField(default=0)
+    n_versions = models.IntegerField(default=0)
+    n_packaged = models.IntegerField(default=0)
+
+class MaintainerLog(models.Model):
+    maintainer = models.ForeignKey(Maintainer)
+    datetime = models.DateTimeField()
+
+    n_packages = models.IntegerField(default=0)
+    n_versions = models.IntegerField(default=0)
+    n_packaged = models.IntegerField(default=0)