euscanwww: big commit

- fix commands
- import DataTables
- initial category view

Signed-off-by: Corentin Chary <corentincj@iksaif.net>

euscanwww/euscan/management/commands/list-packages.py

@@ -0,0 +1,10 @@
+from django.core.management.base import BaseCommand, CommandError
+from euscanwww.euscan.models import Package
+
+class Command(BaseCommand):
+    _overlays = {}
+    help = 'List packages'
+
+    def handle(self, *args, **options):
+        for pkg in Package.objects.all():
+            self.stdout.write('%s/%s\n' % (pkg.category, pkg.name))

euscanwww/euscan/management/commands/scan-portage.py

@@ -141,9 +141,14 @@ class Command(BaseCommand):
         obj, created = Version.objects.get_or_create(package=package, slot=slot,
                                                      revision=rev, version=ver,
                                                      overlay=overlay)
+
+        if created or not package.n_packaged:
+            package.n_packaged += 1
+        if created:
+            package.n_versions += 1
+
+        package.save()
+
         obj.packaged = True
         obj.save()
 
-        package.n_versions += 1
-        package.n_packaged += 1
-        package.save()

euscanwww/euscan/management/commands/scan-upstream.py

@@ -4,6 +4,7 @@ import sys
 import os
 import re
 
+from StringIO import StringIO
 from datetime import datetime
 from portage import versions
 from optparse import make_option
@@ -21,11 +22,11 @@ class Command(BaseCommand):
             dest='all',
             default=False,
             help='Scan all packages'),
-        make_option('--parallel',
+        make_option('--feed',
             action='store_true',
-            dest='parallel',
+            dest='feed',
             default=False,
-            help='Use GNU Parallel'),
+            help='Read euscan output from stdin'),
         make_option('--quiet',
             action='store_true',
             dest='quiet',
@@ -36,9 +37,13 @@ class Command(BaseCommand):
     help = 'Scans metadata and fills database'
 
     def handle(self, *args, **options):
-        if len(args) == 0 and options['all'] == False:
+        if len(args) == 0 and options['all'] == False and options['feed'] == False:
             raise CommandError('You must specify a package or use --all')
 
+        if options['feed']:
+            self.parse_output(options, sys.stdin)
+            return
+
         if not options['quiet']:
             self.stdout.write('Scanning upstream...\n')
 
@@ -57,22 +62,13 @@ class Command(BaseCommand):
 
     @commit_on_success
     def scan(self, options, packages=None):
-        if options['parallel']:
-            jobs = '\n'.join(packages)
-            cmd = ['gparallel', '--jobs', '150%', 'euscan']
+        for package in packages:
+            cmd = ['euscan', package]
 
-            fp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
-            output = fp.communicate(jobs)[0]
+            fp = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+            output = StringIO(fp.communicate()[0])
 
             self.parse_output(options, output)
-        else:
-            for package in packages:
-                cmd = ['euscan', package]
-
-                fp = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-                output = fp.communicate()[0]
-
-                self.parse_output(options, output)
 
     def parse_output(self, options, output):
         from portage.versions import _cp
@@ -83,7 +79,10 @@ class Command(BaseCommand):
         package = None
         log = ""
 
-        for line in output.split('\n'):
+        while True:
+            line = output.readline()
+            if line == '':
+                break
             match = package_re.match(line)
             if match:
                 if package:
@@ -143,5 +142,6 @@ class Command(BaseCommand):
             obj.packaged = False
             obj.save()
 
-        package.n_versions += 1
-        package.save()
+        if created:
+            package.n_versions += 1
+            package.save()

euscanwww/euscan/management/commands/update-counters.py

@@ -0,0 +1,90 @@
+import datetime
+
+from optparse import make_option
+
+from django.db.models import Count, Sum
+from django.db.transaction import commit_on_success
+from django.core.management.base import BaseCommand, CommandError
+from euscanwww.euscan.models import Package, HerdLog, MaintainerLog, CategoryLog, Herd, Maintainer, Version
+
+class Command(BaseCommand):
+    _overlays = {}
+    help = 'Update counters'
+
+    option_list = BaseCommand.option_list + (
+        make_option('--quiet',
+            action='store_true',
+            dest='quiet',
+            default=False,
+            help='Be quiet'),
+        )
+
+    @commit_on_success
+    def handle(self, *args, **options):
+        now = datetime.datetime.now()
+
+        categories = {}
+        herds = {}
+        maintainers = {}
+
+        # Could be done using raw SQL queries, but I don't have time for that
+        # right now ...
+
+        for cat in Package.objects.values('category').distinct():
+            clog = CategoryLog()
+            clog.datetime = now
+            clog.category = cat['category']
+            categories[clog.category] = clog
+
+        for herd in Herd.objects.all():
+            hlog = HerdLog()
+            hlog.datetime = now
+            hlog.herd = herd
+            herds[herd] = hlog
+
+        for maintainer in Maintainer.objects.all():
+            mlog = MaintainerLog()
+            mlog.datetime = now
+            mlog.maintainer = maintainer
+            maintainers[maintainer] = mlog
+
+        for package in Package.objects.all():
+            # Should not be needed, but can't hurt
+            package.n_versions = Version.objects.filter(package=package).count()
+            package.n_packaged != Version.objects.filter(package=package,packaged=True).count()
+            package.save()
+
+            for herd in package.herds.all():
+                herds[herd].n_packages += 1
+                herds[herd].n_versions += package.n_versions
+                herds[herd].n_packaged += package.n_packaged
+
+            for maintainer in package.maintainers.all():
+                maintainers[maintainer].n_packages += 1
+                maintainers[maintainer].n_versions += package.n_versions
+                maintainers[maintainer].n_packaged += package.n_packaged
+
+            categories[package.category].n_packages += 1
+            categories[package.category].n_versions += package.n_versions
+            categories[package.category].n_packaged += package.n_packaged
+
+        for clog in categories.values():
+            if not options['quiet']:
+                self.stdout.write('[c] %s - [%d, %d/%d]\n' %
+                                  (clog.category, clog.n_packages,
+                                   clog.n_packaged, clog.n_versions))
+            clog.save()
+
+        for hlog in herds.values():
+            if not options['quiet']:
+                self.stdout.write('[h] %s - [%d, %d/%d]\n' %
+                                  (hlog.herd, hlog.n_packages,
+                                   hlog.n_packaged, hlog.n_versions))
+            hlog.save()
+
+        for mlog in maintainers.values():
+            if not options['quiet']:
+                self.stdout.write('[m] %s - [%d, %d/%d]\n' %
+                                  (mlog.maintainer, mlog.n_packages,
+                                   mlog.n_packaged, mlog.n_versions))
+            mlog.save()

euscanwww/euscan/templatetags/math_.py

@@ -0,0 +1,29 @@
+from django.template import Node, Library
+import math
+
+register = Library()
+
+# taken from http://lybniz2.sourceforge.net/safeeval.html
+# make a list of safe functions
+math_safe_list = ['acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'cosh', 'degrees', 'e', 'exp', 'fabs', 'floor', 'fmod', 'frexp', 'hypot', 'ldexp', 'log', 'log10', 'modf', 'pi', 'pow', 'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh']
+
+# use the list to filter the local namespace
+math_safe_dict = dict([(k, getattr(math, k)) for k in math_safe_list])
+
+# add any needed builtins back in.
+math_safe_dict['abs'] = abs
+
+@register.filter('math')
+def math_(lopr, expr):
+    """Evals a math expression and returns it's value.
+
+    "$1" is a placeholder. Insert "$1" in the expression where the value is to be used. All math functions such as abs, sin, cos, floor are supported.
+    Example,
+        a. You will be redirected in {{ seconds|math:"$1 / 60.0" }} minutes
+        b. Square of {{ x }} is {{ x|math:"$1 * $1" }}
+        c. Square root of {{ x }} is {{ x|math:"sqrt($1)" }}
+        d. Given x = {{ x }}, (2 + x) * 6 = {{ x|math:"(2 + $1) * 6" }}
+    """
+    if lopr:
+        return eval(expr.replace('$1', str(lopr)), {"__builtins__": None}, math_safe_dict)
+    return ''

euscanwww/euscan/templatetags/sub.py

@@ -0,0 +1,8 @@
+from django import template
+
+register = template.Library()
+
+def sub(value, arg=None):
+    return value-arg
+
+register.filter('sub', sub)

euscanwww/euscan/views.py

@@ -1,9 +1,20 @@
 from annoying.decorators import render_to
 from django.http import Http404
+from django.db.models import Sum, Max
+
+from euscan.models import Version, Package, Herd, Maintainer, EuscanResult
 
 @render_to('euscan/index.html')
 def index(request):
-    return {}
+    ctx = {}
+    ctx['n_packaged'] = Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum']
+    ctx['n_versions'] = Package.objects.aggregate(Sum('n_versions'))['n_versions__sum']
+    ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged']
+    ctx['n_packages'] = Package.objects.count()
+    ctx['n_herds'] = Herd.objects.count()
+    ctx['n_maintainers'] = Maintainer.objects.count()
+    ctx['last_scan'] = EuscanResult.objects.aggregate(Max('datetime'))['datetime__max']
+    return ctx
 
 @render_to('euscan/logs.html')
 def logs(request):
@@ -11,7 +22,8 @@ def logs(request):
 
 @render_to('euscan/categories.html')
 def categories(request):
-    return {}
+    categories = Package.objects.values('category').annotate(n_packaged=Sum('n_packaged'), n_versions=Sum('n_versions'))
+    return { 'categories' : categories }
 
 @render_to('euscan/category.html')
 def category(request, category):