euscanwww: big commit

- fix commands
- import DataTables
- initial category view

Signed-off-by: Corentin Chary <corentincj@iksaif.net>

euscanwww/euscan/management/commands/list-packages.py

@@ -0,0 +1,10 @@
+from django.core.management.base import BaseCommand, CommandError
+from euscanwww.euscan.models import Package
+
+class Command(BaseCommand):
+    _overlays = {}
+    help = 'List packages'
+
+    def handle(self, *args, **options):
+        for pkg in Package.objects.all():
+            self.stdout.write('%s/%s\n' % (pkg.category, pkg.name))

euscanwww/euscan/management/commands/scan-portage.py

@@ -141,9 +141,14 @@ class Command(BaseCommand):
         obj, created = Version.objects.get_or_create(package=package, slot=slot,
                                                      revision=rev, version=ver,
                                                      overlay=overlay)
+
+        if created or not package.n_packaged:
+            package.n_packaged += 1
+        if created:
+            package.n_versions += 1
+
+        package.save()
+
         obj.packaged = True
         obj.save()
 
-        package.n_versions += 1
-        package.n_packaged += 1
-        package.save()

euscanwww/euscan/management/commands/scan-upstream.py

@@ -4,6 +4,7 @@ import sys
 import os
 import re
 
+from StringIO import StringIO
 from datetime import datetime
 from portage import versions
 from optparse import make_option
@@ -21,11 +22,11 @@ class Command(BaseCommand):
             dest='all',
             default=False,
             help='Scan all packages'),
-        make_option('--parallel',
+        make_option('--feed',
             action='store_true',
-            dest='parallel',
+            dest='feed',
             default=False,
-            help='Use GNU Parallel'),
+            help='Read euscan output from stdin'),
         make_option('--quiet',
             action='store_true',
             dest='quiet',
@@ -36,9 +37,13 @@ class Command(BaseCommand):
     help = 'Scans metadata and fills database'
 
     def handle(self, *args, **options):
-        if len(args) == 0 and options['all'] == False:
+        if len(args) == 0 and options['all'] == False and options['feed'] == False:
             raise CommandError('You must specify a package or use --all')
 
+        if options['feed']:
+            self.parse_output(options, sys.stdin)
+            return
+
         if not options['quiet']:
             self.stdout.write('Scanning upstream...\n')
 
@@ -57,22 +62,13 @@ class Command(BaseCommand):
 
     @commit_on_success
     def scan(self, options, packages=None):
-        if options['parallel']:
-            jobs = '\n'.join(packages)
-            cmd = ['gparallel', '--jobs', '150%', 'euscan']
+        for package in packages:
+            cmd = ['euscan', package]
 
-            fp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
-            output = fp.communicate(jobs)[0]
+            fp = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+            output = StringIO(fp.communicate()[0])
 
             self.parse_output(options, output)
-        else:
-            for package in packages:
-                cmd = ['euscan', package]
-
-                fp = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-                output = fp.communicate()[0]
-
-                self.parse_output(options, output)
 
     def parse_output(self, options, output):
         from portage.versions import _cp
@@ -83,7 +79,10 @@ class Command(BaseCommand):
         package = None
         log = ""
 
-        for line in output.split('\n'):
+        while True:
+            line = output.readline()
+            if line == '':
+                break
             match = package_re.match(line)
             if match:
                 if package:
@@ -143,5 +142,6 @@ class Command(BaseCommand):
             obj.packaged = False
             obj.save()
 
-        package.n_versions += 1
-        package.save()
+        if created:
+            package.n_versions += 1
+            package.save()

euscanwww/euscan/management/commands/update-counters.py

@@ -0,0 +1,90 @@
+import datetime
+
+from optparse import make_option
+
+from django.db.models import Count, Sum
+from django.db.transaction import commit_on_success
+from django.core.management.base import BaseCommand, CommandError
+from euscanwww.euscan.models import Package, HerdLog, MaintainerLog, CategoryLog, Herd, Maintainer, Version
+
+class Command(BaseCommand):
+    _overlays = {}
+    help = 'Update counters'
+
+    option_list = BaseCommand.option_list + (
+        make_option('--quiet',
+            action='store_true',
+            dest='quiet',
+            default=False,
+            help='Be quiet'),
+        )
+
+    @commit_on_success
+    def handle(self, *args, **options):
+        now = datetime.datetime.now()
+
+        categories = {}
+        herds = {}
+        maintainers = {}
+
+        # Could be done using raw SQL queries, but I don't have time for that
+        # right now ...
+
+        for cat in Package.objects.values('category').distinct():
+            clog = CategoryLog()
+            clog.datetime = now
+            clog.category = cat['category']
+            categories[clog.category] = clog
+
+        for herd in Herd.objects.all():
+            hlog = HerdLog()
+            hlog.datetime = now
+            hlog.herd = herd
+            herds[herd] = hlog
+
+        for maintainer in Maintainer.objects.all():
+            mlog = MaintainerLog()
+            mlog.datetime = now
+            mlog.maintainer = maintainer
+            maintainers[maintainer] = mlog
+
+        for package in Package.objects.all():
+            # Should not be needed, but can't hurt
+            package.n_versions = Version.objects.filter(package=package).count()
+            package.n_packaged != Version.objects.filter(package=package,packaged=True).count()
+            package.save()
+
+            for herd in package.herds.all():
+                herds[herd].n_packages += 1
+                herds[herd].n_versions += package.n_versions
+                herds[herd].n_packaged += package.n_packaged
+
+            for maintainer in package.maintainers.all():
+                maintainers[maintainer].n_packages += 1
+                maintainers[maintainer].n_versions += package.n_versions
+                maintainers[maintainer].n_packaged += package.n_packaged
+
+            categories[package.category].n_packages += 1
+            categories[package.category].n_versions += package.n_versions
+            categories[package.category].n_packaged += package.n_packaged
+
+        for clog in categories.values():
+            if not options['quiet']:
+                self.stdout.write('[c] %s - [%d, %d/%d]\n' %
+                                  (clog.category, clog.n_packages,
+                                   clog.n_packaged, clog.n_versions))
+            clog.save()
+
+        for hlog in herds.values():
+            if not options['quiet']:
+                self.stdout.write('[h] %s - [%d, %d/%d]\n' %
+                                  (hlog.herd, hlog.n_packages,
+                                   hlog.n_packaged, hlog.n_versions))
+            hlog.save()
+
+        for mlog in maintainers.values():
+            if not options['quiet']:
+                self.stdout.write('[m] %s - [%d, %d/%d]\n' %
+                                  (mlog.maintainer, mlog.n_packages,
+                                   mlog.n_packaged, mlog.n_versions))
+            mlog.save()