Merge branch 'master' of git://github.com/volpino/euscan

Conflicts:
	euscanwww/djeuscan/management/commands/scan_upstream.py

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>

bin/euscan

@@ -21,8 +21,8 @@ __description__ = "A tool to detect new upstream releases."
 import sys
 import os
 import getopt
-import errno
+from errno import EINTR, EINVAL
-import httplib
+from httplib import HTTPConnection
 
 from portage.output import white, yellow, turquoise, green
 from portage.exception import AmbiguousPackageName
@@ -37,6 +37,8 @@ from euscan.out import progress_bar
 
 
 # Globals
+isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty()
+
 
 def exit_helper(status):
     if CONFIG["format"]:
@@ -52,7 +54,7 @@ def setup_signals():
         signal.signal(signal.SIGINT, signal.SIG_IGN)
         signal.signal(signal.SIGTERM, signal.SIG_IGN)
         print()
-        exit_helper(errno.EINTR)
+        exit_helper(EINTR)
 
     signal.signal(signal.SIGINT, exithandler)
     signal.signal(signal.SIGTERM, exithandler)
@@ -121,6 +123,8 @@ def print_usage(_error=None, help=None):
         print(yellow(" -f, --format=<format>") +
               "     - define the output " + yellow("<format>") +
               " (available: json)", file=out)
+        print(yellow(" -p, --progress") +
+              "     - display a progress bar", file=out)
         print(file=out)
 
     if _error in ('packages',) or help:
@@ -172,6 +176,8 @@ def parse_args():
                 CONFIG['format'] = a
                 CONFIG['nocolor'] = True
                 pp.output.nocolor()
+            elif o in ("-p", "--progress"):
+                CONFIG['progress'] = isatty
             else:
                 return_code = False
 
@@ -179,7 +185,7 @@ def parse_args():
 
     # here are the different allowed command line options (getopt args)
     getopt_options = {'short': {}, 'long': {}}
-    getopt_options['short']['global'] = "hVCqv1bf:"
+    getopt_options['short']['global'] = "hVCqv1bf:p"
     getopt_options['long']['global'] = [
         "help", "version", "nocolor", "quiet", "verbose", "oneshot",
         "brute-force=", "format="
@@ -207,7 +213,7 @@ def parse_args():
 def main():
     """Parse command line and execute all actions."""
     CONFIG['nocolor'] = (
-        port_settings["NOCOLOR"] in ('yes', 'true') or not sys.stdout.isatty()
+        port_settings["NOCOLOR"] in ('yes', 'true') or not isatty
     )
     if CONFIG['nocolor']:
         pp.output.nocolor()
@@ -230,19 +236,23 @@ def main():
 
         else:
             print_usage(e.value)
-            exit_helper(errno.EINVAL)
+            exit_helper(EINVAL)
 
     if CONFIG['verbose'] > 2:
-        httplib.HTTPConnection.debuglevel = 1
+        HTTPConnection.debuglevel = 1
 
-    isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty()
+    if not CONFIG["format"]:
+        CONFIG["progress"] = False
 
-    for query in queries:
     on_progress = None
-        if (CONFIG['format'] or CONFIG['quiet']) and isatty:
+    if CONFIG['progress']:
-            print("%s:" % query, file=sys.stderr)
         on_progress_gen = progress_bar()
         on_progress = on_progress_gen.next()
+        on_progress(maxval=len(queries) * 100, increment=0)
+
+    for query in queries:
+        if CONFIG["progress"]:
+            on_progress(increment=10, label=query)
 
         ret = []
 
@@ -271,27 +281,16 @@ def main():
             output.eerror('%s: %s' % (query, str(err)))
             exit_helper(1)
 
-        if not CONFIG['quiet'] and not CONFIG['format']:
+        if not ret and not CONFIG['quiet']:
-            print()
-
-        if (CONFIG['format'] or CONFIG['quiet']) and isatty:
-            on_progress_gen.next()
-            print("\n", file=sys.stderr)
-
-        if ret is not None:
-            if len(ret) > 0:
-                for cp, url, version, handler, confidence in ret:
-                    output.result(cp, version, url, handler, confidence)
-
-                if (CONFIG['format'] or CONFIG['quiet']) and isatty:
-                    print("\n", file=sys.stderr)
-
-            elif not CONFIG['quiet']:
             output.ewarn(
                 "Didn't find any new version, check package's homepage " +
                 "for more informations"
             )
 
+    if CONFIG['progress']:
+        on_progress_gen.next()
+        print("\n", file=sys.stderr)
+
     output.set_query(None)
 
 

euscanwww/README.Celery

@@ -0,0 +1,40 @@
+Requirements
+============
+
+Installing requirements with setup.py is the easiest way, just run::
+
+    python setup.py develop
+
+or::
+
+    python setup.py install  # to install euscan and requirements
+
+If you prefer to use portage just install dev-python/django-celery-2.5.5
+
+There's the need of having a broker for tasks. The default and reccommended
+broker is RabbitMQ.
+Install with::
+emerge rabbitmq-server
+
+
+Running Celery
+==============
+
+You'll need:
+* celeryd (celery daemon for running tasks)::
+
+    python manage.py celeryd -E -l INFO
+
+* celerycam (for monitoring celery and see the results in the django admin page)::
+
+    python manage.py celerycam
+
+* celerybeat (for running periodic tasks)::
+
+    python manage.py celerybeat -l INFO
+
+
+TODO
+====
+
+Provide a supervisord configuration

euscanwww/djeuscan/admin.py

@@ -1,22 +1,54 @@
 from djeuscan.models import Package, Version, VersionLog, EuscanResult, \
-    Log, WorldLog, CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer
+    Log, WorldLog, CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer, \
+    RefreshPackageQuery, HerdAssociation, CategoryAssociation, \
+    MaintainerAssociation, PackageAssociation
 from django.contrib import admin
 
-admin.site.register(Herd)
+
-admin.site.register(Maintainer)
+class EuscanResultAdmin(admin.ModelAdmin):
+    search_fields = ('package__name', 'package__category')
+    list_filter = ('datetime', )
+    ordering = ["-datetime"]
+
+
+class HerdAdmin(admin.ModelAdmin):
+    search_fields = ('herd', 'email')
+    ordering = ["herd"]
+
+
+class MaintainerAdmin(admin.ModelAdmin):
+    search_fields = ('name', 'email')
+    ordering = ["name"]
 
 
 class PackageAdmin(admin.ModelAdmin):
     search_fields = ('category', 'name')
+    list_filter = ('category', )
+
+
+class VersionAdmin(admin.ModelAdmin):
+    search_fields = ('package__name', 'package__category')
+    list_filter = ('overlay', 'packaged', 'alive')
+
 
 admin.site.register(Package, PackageAdmin)
 
-admin.site.register(Version)
+admin.site.register(Herd, HerdAdmin)
+admin.site.register(Maintainer, MaintainerAdmin)
+
+admin.site.register(Version, VersionAdmin)
 admin.site.register(VersionLog)
-admin.site.register(EuscanResult)
+
+admin.site.register(EuscanResult, EuscanResultAdmin)
 
 admin.site.register(Log)
 admin.site.register(WorldLog)
 admin.site.register(CategoryLog)
 admin.site.register(HerdLog)
 admin.site.register(MaintainerLog)
+
+admin.site.register(RefreshPackageQuery)
+admin.site.register(HerdAssociation)
+admin.site.register(CategoryAssociation)
+admin.site.register(MaintainerAssociation)
+admin.site.register(PackageAssociation)

euscanwww/djeuscan/management/commands/regen_rrds.py

@@ -1,25 +1,10 @@
+import logging
 from django.core.management.base import BaseCommand
-from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
-from djeuscan import charts
 
+from djeuscan.processing import set_verbosity_level
+from djeuscan.processing.regen_rrds import regen_rrds
 
-def regen_rrds():
+logger = logging.getLogger(__name__)
-    """
-    Regenerates the rrd database
-    """
-    for wlog in WorldLog.objects.all():
-        charts.rrd_update('world', wlog.datetime, wlog)
-
-    for clog in CategoryLog.objects.all():
-        charts.rrd_update('category-%s' % clog.category,
-                          clog.datetime, clog)
-
-    for hlog in HerdLog.objects.all():
-        charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
-
-    for mlog in MaintainerLog.objects.all():
-        charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
-                          mlog.datetime, mlog)
 
 
 class Command(BaseCommand):
@@ -27,4 +12,5 @@ class Command(BaseCommand):
     help = 'Regenerate rrd database'
 
     def handle(self, *args, **options):
-        regen_rrds()
+        set_verbosity_level(logger, options.get("verbosity", 1))
+        regen_rrds(logger=logger)

euscanwww/djeuscan/management/commands/scan_metadata.py

@@ -1,142 +1,13 @@
 import sys
-
+import logging
 from optparse import make_option
 
-from gentoolkit.query import Query
-from gentoolkit.errors import GentoolkitFatalError
-
-from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand
-from django.core.management.color import color_style
-from django.core.exceptions import ValidationError
 
-from djeuscan.models import Package, Herd, Maintainer
+from djeuscan.processing import set_verbosity_level
+from djeuscan.processing.scan_metadata import scan_metadata
 
-
+logger = logging.getLogger(__name__)
-class ScanMetadata(object):
-    def __init__(self, quiet):
-        self.quiet = quiet
-        self.style = color_style()
-
-    @commit_on_success
-    def run(self, query=None, obj=None):
-        matches = Query(query).find(
-            include_masked=True,
-            in_installed=False,
-        )
-
-        if not matches:
-            sys.stderr.write(
-                self.style.ERROR("Unknown package '%s'\n" % query)
-            )
-            return
-
-        matches = sorted(matches)
-        pkg = matches.pop()
-        if '9999' in pkg.version and len(matches):
-            pkg = matches.pop()
-
-        if not obj:
-            obj, created = Package.objects.get_or_create(
-                category=pkg.category, name=pkg.name
-            )
-        else:
-            created = False
-
-        try:
-            obj.homepage = pkg.environment("HOMEPAGE")
-            obj.description = pkg.environment("DESCRIPTION")
-        except GentoolkitFatalError, err:
-            sys.stderr.write(
-                self.style.ERROR(
-                    "Gentoolkit fatal error: '%s'\n" % str(err)
-                )
-            )
-
-        if created and not self.quiet:
-            sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
-
-        if pkg.metadata:
-            herds = dict(
-                [(herd[0], herd) for herd in pkg.metadata.herds(True)]
-            )
-            maintainers = dict(
-                [(m.email, m) for m in pkg.metadata.maintainers()]
-            )
-
-            existing_herds = [h.herd for h in obj.herds.all()]
-            new_herds = set(herds.keys()).difference(existing_herds)
-            old_herds = set(existing_herds).difference(herds.keys())
-
-            existing_maintainers = [m.email for m in obj.maintainers.all()]
-            new_maintainers = set(
-                maintainers.keys()).difference(existing_maintainers
-            )
-            old_maintainers = set(
-                existing_maintainers).difference(maintainers.keys()
-            )
-
-            for herd in obj.herds.all():
-                if herd.herd in old_herds:
-                    obj.herds.remove(herd)
-
-            for herd in new_herds:
-                herd = self.store_herd(*herds[herd])
-                obj.herds.add(herd)
-
-            for maintainer in obj.maintainers.all():
-                if maintainer.email in old_maintainers:
-                    obj.maintainers.remove(maintainer)
-
-            for maintainer in new_maintainers:
-                maintainer = maintainers[maintainer]
-                try:
-                    maintainer = self.store_maintainer(
-                        maintainer.name, maintainer.email
-                    )
-                    obj.maintainers.add(maintainer)
-                except ValidationError:
-                    sys.stderr.write(
-                        self.style.ERROR("Bad maintainer: '%s' '%s'\n" % (maintainer.name, maintainer.email))
-                    )
-
-        obj.save()
-
-    def store_herd(self, name, email):
-        if not name:
-            name = '{nil}'
-        name = name.strip("\r").strip("\n").strip("\t").strip()
-
-        herd, created = Herd.objects.get_or_create(
-            herd=name,
-            defaults={"email": email}
-        )
-
-        if created and not self.quiet:
-            sys.stdout.write('+ [h] %s <%s>\n' % (name, email))
-
-        herd.email = email
-        herd.save()
-
-        return herd
-
-    def store_maintainer(self, name, email):
-        if not name:
-            name = email
-        if not name:
-            name = '{nil}'
-
-        maintainer, created = Maintainer.objects.get_or_create(
-            email=email,
-            defaults={"name": name}
-        )
-
-        if created:
-            if not self.quiet:
-                sys.stdout.write(
-                    '+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
-                )
-        return maintainer
 
 
 class Command(BaseCommand):
@@ -148,26 +19,19 @@ class Command(BaseCommand):
             dest='all',
             default=False,
             help='Scan all packages'),
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
         )
     args = '<package package ...>'
     help = 'Scans metadata and fills database'
 
     def handle(self, *args, **options):
-        self.options = options
+        set_verbosity_level(logger, options.get("verbosity", 1))
-
-        scan_metadata = ScanMetadata(quiet=options["quiet"])
 
         if options['all']:
-            for pkg in Package.objects.all():
+            packages = None
-                scan_metadata.run('%s/%s' % (pkg.category, pkg.name), pkg)
+
-        elif len(args) > 0:
+        elif len(args):
-            for package in args:
+            packages = [pkg for pkg in args]
-                scan_metadata.run(package)
         else:
-            for package in sys.stdin.readlines():
+            packages = [pkg[:-1] for pkg in sys.stdin.readlines()]
-                scan_metadata.run(package[:-1])
+
+        scan_metadata(packages=packages, logger=logger)

euscanwww/djeuscan/management/commands/scan_portage.py

@@ -1,274 +1,13 @@
-import subprocess
-import portage
 import sys
-import os
+import logging
-import re
-
 from optparse import make_option
 
-from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand
-from django.core.management.color import color_style
 
-from djeuscan.models import Package, Version, VersionLog
+from djeuscan.processing import set_verbosity_level
+from djeuscan.processing.scan_portage import scan_portage
 
-
+logger = logging.getLogger(__name__)
-class ScanPortage(object):
-    def __init__(self, stdout=None, **options):
-        if stdout is None:
-            self.stdout = sys.stdout
-        else:
-            self.stdout = stdout
-
-        self.options = options
-        self.style = color_style()
-        self._cache = {'packages': {}, 'versions': {}}
-        self._overlays = None
-
-    def cache_hash_package(self, category, name):
-        return '%s/%s' % (category, name)
-
-    def cache_store_package(self, package):
-        key = self.cache_hash_package(package.category, package.name)
-        self._cache['packages'][key] = package
-
-    def cache_get_package(self, category, name):
-        return self._cache['packages'].get(
-            self.cache_hash_package(category, name)
-        )
-
-    def cache_hash_version(self, category, name, version, revision, slot,
-                           overlay):
-        key = '%s/%s-%s-r%s %s %s' % (category, name,
-                                      version, revision,
-                                      slot, overlay)
-        return key
-
-    def cache_get_version(self, category, name, version, revision, slot,
-                          overlay):
-        key = self.cache_hash_version(category, name, version, revision, slot,
-                                      overlay)
-        return self._cache['versions'].get(key)
-
-    def cache_store_version(self, version):
-        key = self.cache_hash_version(
-            version.package.category, version.package.name, version.version,
-            version.revision, version.slot, version.overlay
-        )
-        self._cache['versions'][key] = version
-
-    def overlays(self):
-        if self._overlays:
-            return self._overlays
-
-        env = os.environ
-        env['OVERLAYS_LIST'] = 'all'
-        env['PRINT_COUNT_ALWAYS'] = 'never'
-
-        cmd = ['eix', '-!']
-
-        output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
-            communicate()[0]
-        output = output.strip().strip('\n').split('\n')
-
-        overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
-
-        self._overlays = {}
-
-        for line in output:
-            match = overlay_re.match(line)
-            if not match:
-                continue
-            self._overlays[match.group('key')] = match.group('name')
-
-        return self._overlays
-
-    @commit_on_success
-    def run(self, query=None):
-        env = os.environ
-        env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
-
-        cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages',
-               '-x']
-        if query:
-            cmd.extend(['--exact', query])
-
-        if self.options['all']:
-            if not self.options['quiet']:
-                self.stdout.write('Killing existing versions...')
-                self.stdout.flush()
-            Version.objects.filter(packaged=True).update(alive=False)
-            if not self.options['quiet']:
-                self.stdout.write('done\n')
-
-        output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
-            communicate()[0]
-        output = output.strip().strip('\n')
-
-        if len(output) == 0:
-            if not query:
-                return
-            if self.options['purge-packages']:
-                if not self.options['quiet']:
-                    sys.stdout.write('- [p] %s\n' % (query))
-                if '/' in query:
-                    cat, pkg = portage.catsplit(query)
-                    Package.objects.filter(category=cat, name=pkg).delete()
-                else:
-                    Package.objects.filter(name=query).delete()
-            else:
-                sys.stderr.write(
-                    self.style.ERROR(
-                        "Unknown package '%s'\n" % query
-                    )
-                )
-            return
-
-        output = output.split('\n')
-        packages = {}
-
-        line_re = re.compile(
-            r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$'
-        )
-
-        package = None
-
-        for line in output:
-            match = line_re.match(line)
-
-            if not match:
-                continue
-
-            cpv = match.group('cpv')
-            slot = match.group('slot')
-            overlay = match.group('overlay')
-
-            cat, pkg, ver, rev = portage.catpkgsplit(cpv)
-
-            packages['%s/%s' % (cat, pkg)] = True
-
-            if not package or not \
-               (cat == package.category and pkg == package.name):
-                package = self.store_package(cat, pkg)
-
-            self.store_version(package, cpv, slot, overlay)
-
-        if self.options['purge-packages'] and not query:
-            for package in Package.objects.all():
-                cp = "%s/%s" % (package.category, package.name)
-                if cp not in packages:
-                    if not self.options['quiet']:
-                        sys.stdout.write('- [p] %s\n' % (package))
-                    package.delete()
-
-    def store_package(self, cat, pkg):
-        created = False
-        obj = self.cache_get_package(cat, pkg)
-
-        if not obj:
-            obj, created = Package.objects.get_or_create(
-                category=cat,
-                name=pkg
-            )
-            self.cache_store_package(obj)
-
-        if created:
-            if not self.options['quiet']:
-                sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
-
-        # Set all versions dead, then set found versions alive and
-        # delete old versions
-        if not self.options['all']:
-            Version.objects.filter(
-                package=obj,
-                packaged=True
-            ).update(alive=False)
-
-        return obj
-
-    def store_version(self, package, cpv, slot, overlay):
-        cat, pkg, ver, rev = portage.catpkgsplit(cpv)
-
-        overlays = self.overlays()
-
-        if overlay in overlays:
-            overlay = overlays[overlay]
-        else:
-            overlay = 'gentoo'
-
-        created = False
-        obj = self.cache_get_version(
-            package.category, package.name, ver, rev, slot, overlay
-        )
-        if not obj:
-            obj, created = Version.objects.get_or_create(
-                package=package, slot=slot,
-                revision=rev, version=ver,
-                overlay=overlay,
-                defaults={"alive": True, "packaged": True}
-            )
-        if not created: # Created objects have defaults values
-            obj.alive = True
-            obj.packaged = True
-            obj.save()
-
-        if created:
-            self.cache_store_version(obj)
-
-        # nothing to do (note: it can't be an upstream version because
-        # overlay can't be empty here)
-        if not created:
-            return
-
-        if not self.options['quiet']:
-            sys.stdout.write('+ [v] %s \n' % (obj))
-
-        if overlay == 'gentoo':
-            package.n_packaged += 1
-        else:
-            package.n_overlay += 1
-        package.n_versions += 1
-        package.save()
-
-        if self.options['no-log']:
-            return
-
-        VersionLog.objects.create(
-            package=obj.package,
-            action=VersionLog.VERSION_ADDED,
-            slot=obj.slot,
-            revision=obj.revision,
-            version=obj.version,
-            overlay=obj.overlay
-        )
-
-@commit_on_success
-def purge_versions(options):
-    # For each dead versions
-    for version in Version.objects.filter(packaged=True, alive=False):
-        if version.overlay == 'gentoo':
-            version.package.n_packaged -= 1
-        else:
-            version.package.n_overlay -= 1
-        version.package.n_versions -= 1
-        version.package.save()
-
-        if not options['quiet']:
-            sys.stdout.write('- [v] %s\n' % (version))
-
-        if options['no-log']:
-            continue
-
-        VersionLog.objects.create(
-            package=version.package,
-            action=VersionLog.VERSION_REMOVED,
-            slot=version.slot,
-            revision=version.revision,
-            version=version.version,
-            overlay=version.overlay
-        )
-
-    Version.objects.filter(packaged=True, alive=False).delete()
 
 
 class Command(BaseCommand):
@@ -301,43 +40,26 @@ class Command(BaseCommand):
             default=False,
             help=('Prefetch all versions and packages from DB to '
                   'speedup full scan process.')),
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
         )
     args = '[package package ...]'
     help = 'Scans portage tree and fills database'
 
     def handle(self, *args, **options):
-        scan_portage = ScanPortage(stdout=self.stdout, **options)
+        set_verbosity_level(logger, options.get("verbosity", 1))
-
-        if not options['quiet']:
-            self.stdout.write('Scanning portage tree...\n')
-
-        if options['prefetch']:
-            if not options['quiet']:
-                self.stdout.write('Prefetching objects...')
-                self.stdout.flush()
-            for package in Package.objects.all():
-                scan_portage.cache_store_package(package)
-            for version in Version.objects.select_related('package').all():
-                scan_portage.cache_store_version(version)
-            if not options['quiet']:
-                self.stdout.write('done\n')
 
         if options['all']:
-            scan_portage.run()
+            packages = None
+
         elif len(args):
-            for package in args:
+            packages = [pkg for pkg in args]
-                scan_portage.run(package)
         else:
-            for package in sys.stdin.readlines():
+            packages = [pkg[:-1] for pkg in sys.stdin.readlines()]
-                scan_portage.run(package[:-1])
 
-        if options['purge-versions']:
+        scan_portage(
-            purge_versions(options)
+            packages=packages,
-
+            no_log=options["no-log"],
-        if not options['quiet']:
+            purge_packages=options["purge-packages"],
-            self.stdout.write('Done.\n')
+            purge_versions=options["purge-versions"],
+            prefetch=options["prefetch"],
+            logger=logger,
+        )

euscanwww/djeuscan/management/commands/scan_upstream.py

@@ -1,151 +1,13 @@
-import subprocess
+import logging
-import portage
 import sys
-import re
-from StringIO import StringIO
 from optparse import make_option
-from collections import defaultdict
 
-from django.utils import timezone
-from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand
 
-from djeuscan.models import Package, Version, EuscanResult, VersionLog
+from djeuscan.processing import set_verbosity_level
+from djeuscan.processing.scan_upstream import scan_upstream
 
-
+logger = logging.getLogger(__name__)
-class ScanUpstream(object):
-    def __init__(self, options=None):
-        if options is None:
-            self.options = defaultdict(None)
-        else:
-            self.options = options
-
-    def run(self, packages=None):
-        for package in packages:
-            cmd = ['euscan', package]
-
-            fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                                  stderr=subprocess.PIPE)
-            output = StringIO(fp.communicate()[0])
-
-            self.parse_output(output)
-
-    def store_result(self, package, log):
-        # Remove previous logs
-        EuscanResult.objects.filter(package=package).delete()
-
-        obj = EuscanResult()
-        obj.package = package
-        obj.result = log
-        obj.datetime = timezone.now()
-        obj.save()
-
-    def store_package(self, cpv):
-        cat, pkg, ver, rev = portage.catpkgsplit(cpv)
-
-        obj, created = Package.objects.get_or_create(category=cat, name=pkg)
-
-        if created and not self.options['quiet']:
-            sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
-
-        # Set all versions dead, then set found versions alive and
-        # delete old versions
-        Version.objects.filter(package=obj, packaged=False).update(alive=False)
-
-        return obj
-
-    def store_version(self, package, ver, url):
-        obj, created = Version.objects.get_or_create(
-            package=package, slot='', revision='r0', version=ver, overlay='',
-            defaults={"alive": True, "urls": url, "packaged": False}
-        )
-        if not created:
-            obj.alive = True
-            obj.urls = url
-            obj.packaged = False
-            obj.save()
-
-        # If it's not a new version, just update the object and continue
-        if not created:
-            return
-
-        if not self.options['quiet']:
-            sys.stdout.write('+ [u] %s %s\n' % (obj, url))
-
-        VersionLog.objects.create(
-            package=package,
-            action=VersionLog.VERSION_ADDED,
-            slot='',
-            revision='r0',
-            version=ver,
-            overlay=''
-        )
-
-        package.n_versions += 1
-        package.save()
-
-    @commit_on_success
-    def parse_output(self, output):
-        from portage.versions import _cp
-
-        if type(_cp) == dict:
-            _cp = _cp["dots_allowed_in_PN"]
-
-        package_re = re.compile(
-            r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$'
-        )
-        version_re = re.compile(
-            r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$'
-        )
-
-        package = None
-        log = ""
-
-        while True:
-            line = output.readline()
-            if line == '':
-                break
-            match = package_re.match(line)
-            if match:
-                if package:
-                    self.store_result(package, log)
-
-                cpv = match.group('cpv')
-                package = self.store_package(cpv)
-                log = line
-                continue
-
-            log += line
-
-            match = version_re.match(line)
-            if match:
-                ver = match.group('ver')
-                url = match.group('url')
-                self.store_version(package, ver, url)
-
-        if package:
-            self.store_result(package, log)
-
-
-@commit_on_success
-def purge_versions(options):
-    # For each dead versions
-    for version in Version.objects.filter(packaged=False, alive=False):
-        VersionLog.objects.create(
-            package=version.package,
-            action=VersionLog.VERSION_REMOVED,
-            slot=version.slot,
-            revision=version.revision,
-            version=version.version,
-            overlay=version.overlay
-        )
-
-        version.package.n_versions -= 1
-        version.package.save()
-
-        if not options['quiet']:
-            sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
-    Version.objects.filter(packaged=False, alive=False).delete()
 
 
 class Command(BaseCommand):
@@ -162,41 +24,23 @@ class Command(BaseCommand):
             dest='purge-versions',
             default=False,
             help='Purge old versions'),
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
         )
     args = '<package package ...>'
     help = 'Scans metadata and fills database'
 
     def handle(self, *args, **options):
-        scan_upstream = ScanUpstream(options)
+        set_verbosity_level(logger, options.get("verbosity", 1))
-
-        if not args and not options['all']:
-            scan_upstream.parse_output(sys.stdin)
-            if options['purge-versions']:
-                purge_versions(options)
-            return
-
-        if not options['quiet']:
-            self.stdout.write('Scanning upstream...\n')
-
-        packages = []
 
         if options['all']:
-            for pkg in Package.objects.all():
+            packages = None
-                packages.append('%s/%s' % (pkg.category, pkg.name))
+
-        elif args:
+        elif len(args):
-            packages = list(args)
+            packages = [pkg for pkg in args]
         else:
-            packages = [package[:-1] for package in sys.stdin.readlines()]
+            packages = [pkg[:-1] for pkg in sys.stdin.readlines()]
 
-        scan_upstream.run(packages)
+        scan_upstream(
-
+            packages=packages,
-        if options['purge-versions']:
+            purge_versions=options["purge-versions"],
-            purge_versions(options)
+            logger=logger,
-
+        )
-        if not options['quiet']:
-            self.stdout.write('Done.\n')

euscanwww/djeuscan/management/commands/update_counters.py

@@ -1,185 +1,12 @@
-import sys
+import logging
 from optparse import make_option
 
-from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand
-from django.utils import timezone
 
-from djeuscan.models import Package, Herd, Maintainer, Version
+from djeuscan.processing import set_verbosity_level
-from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
+from djeuscan.processing.update_counters import update_counters
-from djeuscan import charts
 
-from distutils.version import StrictVersion, LooseVersion
+logger = logging.getLogger(__name__)
-
-
-def compare_versions(version1, version2):
-    try:
-        return cmp(StrictVersion(version1), StrictVersion(version2))
-    # in case of abnormal version number, fall back to LooseVersion
-    except ValueError:
-        return cmp(LooseVersion(version1), LooseVersion(version2))
-
-
-def add_safe(storage, key):
-    if key not in storage:
-        storage[key] = 1
-    else:
-        storage[key] += 1
-
-
-def add_last_ver(storage, version):
-    key = version['package_id']
-    if key not in storage:
-        storage[key] = version
-        return
-    if version['version'].startswith('9999'):
-        return
-    if compare_versions(storage[key]['version'],
-                        version['version']) < 0:
-        storage[key] = version
-
-
-@commit_on_success
-def update_counters(stdout=None, **options):
-    if stdout is None:
-        stdout = sys.stdout
-
-    now = timezone.now()
-
-    categories = {}
-    herds = {}
-    maintainers = {}
-
-    wlog = None
-
-    if not options['nolog']:
-        wlog = WorldLog()
-        wlog.datetime = now
-
-        for cat in Package.objects.values('category').distinct():
-            clog = CategoryLog()
-            clog.datetime = now
-            clog.category = cat['category']
-            categories[clog.category] = clog
-
-        for herd in Herd.objects.all():
-            hlog = HerdLog()
-            hlog.datetime = now
-            hlog.herd = herd
-            herds[herd.id] = hlog
-
-        for maintainer in Maintainer.objects.all():
-            mlog = MaintainerLog()
-            mlog.datetime = now
-            mlog.maintainer = maintainer
-            maintainers[maintainer.id] = mlog
-
-    package_queryset = Package.objects.all()
-
-    n_versions = {}
-    n_packaged = {}
-    n_overlay = {}
-
-    last_versions_gentoo = {}
-    last_versions_overlay = {}
-    last_versions_upstream = {}
-
-    if not options['fast']:
-        attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
-        for version in Version.objects.all().values(*attrs):
-            overlay, packaged = version['overlay'], version['packaged']
-            package_id = version['package_id']
-
-            add_safe(n_versions, package_id)
-
-            if not packaged:
-                add_last_ver(last_versions_upstream, version)
-                continue
-            if overlay == 'gentoo':
-                add_safe(n_packaged, package_id)
-                add_last_ver(last_versions_gentoo, version)
-            else:
-                add_safe(n_overlay, package_id)
-                add_last_ver(last_versions_overlay, version)
-
-    for package in package_queryset.select_related('herds', 'maintainers'):
-        if not options['fast']:
-            package.n_versions = n_versions.get(package.id, 0)
-            package.n_packaged = n_packaged.get(package.id, 0)
-            package.n_overlay = n_overlay.get(package.id, 0)
-
-            default = {'id': None}
-            package.last_version_gentoo_id = last_versions_gentoo.get(
-                package.id, default
-            )['id']
-            package.last_version_overlay_id = last_versions_overlay.get(
-                package.id, default
-            )['id']
-            package.last_version_upstream_id = last_versions_upstream.get(
-                package.id, default
-            )['id']
-
-            package.save()
-
-        n_packages_gentoo = int(package.n_packaged == package.n_versions)
-        n_packages_overlay = int(package.n_overlay and package.n_packaged \
-                                 + package.n_overlay == package.n_versions)
-        n_packages_outdated = int(package.n_packaged + package.n_overlay \
-                                  < package.n_versions)
-
-        def update_row(storage, key):
-            storage[key].n_packages_gentoo += n_packages_gentoo
-            storage[key].n_packages_overlay += n_packages_overlay
-            storage[key].n_packages_outdated += n_packages_outdated
-
-            storage[key].n_versions_gentoo += package.n_packaged
-            storage[key].n_versions_overlay += package.n_overlay
-            storage[key].n_versions_upstream += package.n_versions - \
-                                                package.n_packaged - \
-                                                package.n_overlay
-
-        def update_log(storage, qs):
-            for row in qs:
-                update_row(storage, row['id'])
-
-        if not options['nolog']:
-            update_log(herds, package.herds.all().values('id'))
-            update_log(maintainers, package.maintainers.all().values('id'))
-            update_row(categories, package.category)
-
-            wlog.n_packages_gentoo += n_packages_gentoo
-            wlog.n_packages_overlay += n_packages_overlay
-            wlog.n_packages_outdated += n_packages_outdated
-
-            wlog.n_versions_gentoo += package.n_packaged
-            wlog.n_versions_overlay += package.n_overlay
-            wlog.n_versions_upstream += package.n_versions - \
-                                        package.n_packaged - \
-                                        package.n_overlay
-
-    if options['nolog']:
-        return
-
-    for clog in categories.values():
-        if not options['quiet']:
-            stdout.write('+ [cl] %s\n' % clog)
-        charts.rrd_update('category-%s' % clog.category, now, clog)
-        clog.save()
-
-    for hlog in herds.values():
-        if not options['quiet']:
-            stdout.write('+ [hl] %s\n' % hlog)
-        charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
-        hlog.save()
-
-    for mlog in maintainers.values():
-        if not options['quiet']:
-            stdout.write('+ [ml] %s\n' % mlog)
-        charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
-        mlog.save()
-
-    charts.rrd_update('world', now, wlog)
-    wlog.save()
 
 
 class Command(BaseCommand):
@@ -187,11 +14,6 @@ class Command(BaseCommand):
     help = 'Update counters'
 
     option_list = BaseCommand.option_list + (
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
         make_option('--fast',
             action='store_true',
             dest='fast',
@@ -205,4 +27,9 @@ class Command(BaseCommand):
     )
 
     def handle(self, *args, **options):
-        update_counters(stdout=self.stdout, **options)
+        set_verbosity_level(logger, options.get("verbosity", 1))
+        update_counters(
+            fast=options["fast"],
+            nolog=options["nolog"],
+            logger=logger,
+        )

euscanwww/djeuscan/managers.py

@@ -35,7 +35,6 @@ ANNOTATE_DICT = {name: models.Sum(name)
 
 
 class PackageMixin(object):
-
     for_maintainer = _gen_for_function('maintainers')
     for_herd = _gen_for_function('herds')
     for_category = _gen_for_function('category')

euscanwww/djeuscan/migrations/0003_auto__add_refreshpackagequery__chg_field_package_last_version_gentoo__.py

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'RefreshPackageQuery'
+        db.create_table('djeuscan_refreshpackagequery', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('query', self.gf('django.db.models.fields.CharField')(unique=True, max_length=256)),
+            ('priority', self.gf('django.db.models.fields.IntegerField')(default=0)),
+        ))
+        db.send_create_signal('djeuscan', ['RefreshPackageQuery'])
+
+
+        # Changing field 'Package.last_version_gentoo'
+        db.alter_column('djeuscan_package', 'last_version_gentoo_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['djeuscan.Version']))
+
+        # Changing field 'Package.last_version_overlay'
+        db.alter_column('djeuscan_package', 'last_version_overlay_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['djeuscan.Version']))
+
+        # Changing field 'Package.last_version_upstream'
+        db.alter_column('djeuscan_package', 'last_version_upstream_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['djeuscan.Version']))
+    def backwards(self, orm):
+        # Deleting model 'RefreshPackageQuery'
+        db.delete_table('djeuscan_refreshpackagequery')
+
+
+        # Changing field 'Package.last_version_gentoo'
+        db.alter_column('djeuscan_package', 'last_version_gentoo_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['djeuscan.Version']))
+
+        # Changing field 'Package.last_version_overlay'
+        db.alter_column('djeuscan_package', 'last_version_overlay_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['djeuscan.Version']))
+
+        # Changing field 'Package.last_version_upstream'
+        db.alter_column('djeuscan_package', 'last_version_upstream_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['djeuscan.Version']))
+    models = {
+        'djeuscan.categorylog': {
+            'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.euscanresult': {
+            'Meta': {'object_name': 'EuscanResult'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'djeuscan.herd': {
+            'Meta': {'object_name': 'Herd'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
+            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'djeuscan.herdlog': {
+            'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.log': {
+            'Meta': {'object_name': 'Log'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'djeuscan.maintainer': {
+            'Meta': {'object_name': 'Maintainer'},
+            'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.maintainerlog': {
+            'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
+        },
+        'djeuscan.package': {
+            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
+            'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.refreshpackagequery': {
+            'Meta': {'object_name': 'RefreshPackageQuery'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
+        },
+        'djeuscan.version': {
+            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
+            'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.versionlog': {
+            'Meta': {'object_name': 'VersionLog'},
+            'action': ('django.db.models.fields.IntegerField', [], {}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.worldlog': {
+            'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        }
+    }
+
+    complete_apps = ['djeuscan']

euscanwww/djeuscan/migrations/0004_auto__add_categoryassociation__add_unique_categoryassociation_user_cat.py

@@ -0,0 +1,238 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'CategoryAssociation'
+        db.create_table('djeuscan_categoryassociation', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('category', self.gf('django.db.models.fields.CharField')(max_length=128)),
+        ))
+        db.send_create_signal('djeuscan', ['CategoryAssociation'])
+
+        # Adding unique constraint on 'CategoryAssociation', fields ['user', 'category']
+        db.create_unique('djeuscan_categoryassociation', ['user_id', 'category'])
+
+        # Adding model 'PackageAssociation'
+        db.create_table('djeuscan_packageassociation', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('package', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Package'])),
+        ))
+        db.send_create_signal('djeuscan', ['PackageAssociation'])
+
+        # Adding unique constraint on 'PackageAssociation', fields ['user', 'package']
+        db.create_unique('djeuscan_packageassociation', ['user_id', 'package_id'])
+
+        # Adding model 'HerdAssociation'
+        db.create_table('djeuscan_herdassociation', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('herd', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Herd'])),
+        ))
+        db.send_create_signal('djeuscan', ['HerdAssociation'])
+
+        # Adding unique constraint on 'HerdAssociation', fields ['user', 'herd']
+        db.create_unique('djeuscan_herdassociation', ['user_id', 'herd_id'])
+
+        # Adding model 'MaintainerAssociation'
+        db.create_table('djeuscan_maintainerassociation', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('maintainer', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Maintainer'])),
+        ))
+        db.send_create_signal('djeuscan', ['MaintainerAssociation'])
+
+        # Adding unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer']
+        db.create_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id'])
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer']
+        db.delete_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id'])
+
+        # Removing unique constraint on 'HerdAssociation', fields ['user', 'herd']
+        db.delete_unique('djeuscan_herdassociation', ['user_id', 'herd_id'])
+
+        # Removing unique constraint on 'PackageAssociation', fields ['user', 'package']
+        db.delete_unique('djeuscan_packageassociation', ['user_id', 'package_id'])
+
+        # Removing unique constraint on 'CategoryAssociation', fields ['user', 'category']
+        db.delete_unique('djeuscan_categoryassociation', ['user_id', 'category'])
+
+        # Deleting model 'CategoryAssociation'
+        db.delete_table('djeuscan_categoryassociation')
+
+        # Deleting model 'PackageAssociation'
+        db.delete_table('djeuscan_packageassociation')
+
+        # Deleting model 'HerdAssociation'
+        db.delete_table('djeuscan_herdassociation')
+
+        # Deleting model 'MaintainerAssociation'
+        db.delete_table('djeuscan_maintainerassociation')
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'djeuscan.categoryassociation': {
+            'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.categorylog': {
+            'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.euscanresult': {
+            'Meta': {'object_name': 'EuscanResult'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'djeuscan.herd': {
+            'Meta': {'object_name': 'Herd'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
+            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'djeuscan.herdassociation': {
+            'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.herdlog': {
+            'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.log': {
+            'Meta': {'object_name': 'Log'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'djeuscan.maintainer': {
+            'Meta': {'object_name': 'Maintainer'},
+            'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.maintainerassociation': {
+            'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.maintainerlog': {
+            'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
+        },
+        'djeuscan.package': {
+            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
+            'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.packageassociation': {
+            'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.refreshpackagequery': {
+            'Meta': {'object_name': 'RefreshPackageQuery'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
+        },
+        'djeuscan.version': {
+            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
+            'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.versionlog': {
+            'Meta': {'object_name': 'VersionLog'},
+            'action': ('django.db.models.fields.IntegerField', [], {}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.worldlog': {
+            'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        }
+    }
+
+    complete_apps = ['djeuscan']

euscanwww/djeuscan/migrations/0005_auto__add_field_version_version_type__add_field_version_handler__add_f.py

@@ -0,0 +1,197 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'Version.version_type'
+        db.add_column('djeuscan_version', 'version_type',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True),
+                      keep_default=False)
+
+        # Adding field 'Version.handler'
+        db.add_column('djeuscan_version', 'handler',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True),
+                      keep_default=False)
+
+        # Adding field 'Version.confidence'
+        db.add_column('djeuscan_version', 'confidence',
+                      self.gf('django.db.models.fields.IntegerField')(default=0),
+                      keep_default=False)
+
+    def backwards(self, orm):
+        # Deleting field 'Version.version_type'
+        db.delete_column('djeuscan_version', 'version_type')
+
+        # Deleting field 'Version.handler'
+        db.delete_column('djeuscan_version', 'handler')
+
+        # Deleting field 'Version.confidence'
+        db.delete_column('djeuscan_version', 'confidence')
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'djeuscan.categoryassociation': {
+            'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.categorylog': {
+            'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.euscanresult': {
+            'Meta': {'object_name': 'EuscanResult'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'djeuscan.herd': {
+            'Meta': {'object_name': 'Herd'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
+            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'djeuscan.herdassociation': {
+            'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.herdlog': {
+            'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.log': {
+            'Meta': {'object_name': 'Log'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'djeuscan.maintainer': {
+            'Meta': {'object_name': 'Maintainer'},
+            'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.maintainerassociation': {
+            'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.maintainerlog': {
+            'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
+        },
+        'djeuscan.package': {
+            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
+            'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.packageassociation': {
+            'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.refreshpackagequery': {
+            'Meta': {'object_name': 'RefreshPackageQuery'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
+        },
+        'djeuscan.version': {
+            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
+            'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'version_type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
+        },
+        'djeuscan.versionlog': {
+            'Meta': {'object_name': 'VersionLog'},
+            'action': ('django.db.models.fields.IntegerField', [], {}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.worldlog': {
+            'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        }
+    }
+
+    complete_apps = ['djeuscan']

euscanwww/djeuscan/migrations/0006_auto__add_field_euscanresult_scan_time__add_field_euscanresult_ebuild.py

@@ -0,0 +1,191 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'EuscanResult.scan_time'
+        db.add_column('djeuscan_euscanresult', 'scan_time',
+                      self.gf('django.db.models.fields.FloatField')(null=True, blank=True),
+                      keep_default=False)
+
+        # Adding field 'EuscanResult.ebuild'
+        db.add_column('djeuscan_euscanresult', 'ebuild',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=256, blank=True),
+                      keep_default=False)
+
+    def backwards(self, orm):
+        # Deleting field 'EuscanResult.scan_time'
+        db.delete_column('djeuscan_euscanresult', 'scan_time')
+
+        # Deleting field 'EuscanResult.ebuild'
+        db.delete_column('djeuscan_euscanresult', 'ebuild')
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'djeuscan.categoryassociation': {
+            'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.categorylog': {
+            'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.euscanresult': {
+            'Meta': {'object_name': 'EuscanResult'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'djeuscan.herd': {
+            'Meta': {'object_name': 'Herd'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
+            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'djeuscan.herdassociation': {
+            'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.herdlog': {
+            'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.log': {
+            'Meta': {'object_name': 'Log'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'djeuscan.maintainer': {
+            'Meta': {'object_name': 'Maintainer'},
+            'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.maintainerassociation': {
+            'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.maintainerlog': {
+            'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
+        },
+        'djeuscan.package': {
+            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
+            'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.packageassociation': {
+            'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.refreshpackagequery': {
+            'Meta': {'object_name': 'RefreshPackageQuery'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
+        },
+        'djeuscan.version': {
+            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
+            'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'version_type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
+        },
+        'djeuscan.versionlog': {
+            'Meta': {'object_name': 'VersionLog'},
+            'action': ('django.db.models.fields.IntegerField', [], {}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.worldlog': {
+            'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        }
+    }
+
+    complete_apps = ['djeuscan']

euscanwww/djeuscan/models.py

@@ -2,6 +2,8 @@ from django.db import models
 from django.core.validators import RegexValidator, validate_email, URLValidator
 from django.core.exceptions import ValidationError
 
+from django.contrib.auth.models import User
+
 from djeuscan.managers import PackageManager, VersionLogManager, \
     EuscanResultManager
 
@@ -115,16 +117,19 @@ class Version(models.Model):
     """
 
     package = models.ForeignKey(Package)
-    slot = models.CharField(max_length=128, blank=True, default='')
+    slot = models.CharField(max_length=128, blank=True, default="")
     revision = models.CharField(max_length=128)
     version = models.CharField(max_length=128)
     packaged = models.BooleanField()
-    overlay = models.CharField(max_length=128, blank=True,
+    overlay = models.CharField(max_length=128, default='gentoo', db_index=True,
-                               default='gentoo', db_index=True,
+                               validators=[validate_name], blank=True)
-                               validators=[validate_name])
     urls = models.TextField(blank=True)
     alive = models.BooleanField(default=True, db_index=True)
 
+    version_type = models.CharField(max_length=128, blank=True)
+    handler = models.CharField(max_length=128, blank=True)
+    confidence = models.IntegerField(default=0)
+
     class Meta:
         unique_together = ['package', 'slot', 'revision', 'version', 'overlay']
 
@@ -149,12 +154,12 @@ class VersionLog(models.Model):
 
     package = models.ForeignKey(Package)
     datetime = models.DateTimeField(auto_now_add=True)
-    slot = models.CharField(max_length=128, blank=True, default='')
+    slot = models.CharField(max_length=128, blank=True, default="")
     revision = models.CharField(max_length=128)
     version = models.CharField(max_length=128)
     packaged = models.BooleanField()
-    overlay = models.CharField(max_length=128, blank=True,
+    overlay = models.CharField(max_length=128, default='gentoo',
-                               default='gentoo', validators=[validate_name])
+                               validators=[validate_name], blank=True)
     action = models.IntegerField(choices=VERSION_ACTIONS)
 
     objects = VersionLogManager()
@@ -182,6 +187,9 @@ class EuscanResult(models.Model):
     datetime = models.DateTimeField()
     result = models.TextField(blank=True)
 
+    scan_time = models.FloatField(null=True, blank=True)
+    ebuild = models.CharField(blank=True, max_length=256)
+
     objects = EuscanResultManager()
 
     class Meta:
@@ -191,6 +199,11 @@ class EuscanResult(models.Model):
         self.full_clean()
         super(EuscanResult, self).save(*args, **kwargs)
 
+    def __unicode__(self):
+        return '[%s] %s/%s' % (
+            self.datetime, self.package.category, self.package.name
+        )
+
 
 class Log(models.Model):
     """
@@ -253,3 +266,55 @@ class MaintainerLog(Log):
 
     def __unicode__(self):
         return u'%s %s' % (self.maintainer, Log.__unicode__(self))
+
+
+class RefreshPackageQuery(models.Model):
+    query = models.CharField(max_length=256, unique=True)
+    priority = models.IntegerField(default=0)
+
+    def __unicode__(self):
+        return u'[%d] %s' % (self.priority, self.query)
+
+
+class HerdAssociation(models.Model):
+    user = models.ForeignKey(User)
+    herd = models.ForeignKey(Herd)
+
+    class Meta:
+        unique_together = ['user', 'herd']
+
+    def __unicode__(self):
+        return u'[%s] %s' % (self.user, self.herd)
+
+
+class MaintainerAssociation(models.Model):
+    user = models.ForeignKey(User)
+    maintainer = models.ForeignKey(Maintainer)
+
+    class Meta:
+        unique_together = ['user', 'maintainer']
+
+    def __unicode__(self):
+        return u'[%s] %s' % (self.user, self.maintainer)
+
+
+class PackageAssociation(models.Model):
+    user = models.ForeignKey(User)
+    package = models.ForeignKey(Package)
+
+    class Meta:
+        unique_together = ['user', 'package']
+
+    def __unicode__(self):
+        return u'[%s] %s' % (self.user, self.package)
+
+
+class CategoryAssociation(models.Model):
+    user = models.ForeignKey(User)
+    category = models.CharField(max_length=128, validators=[validate_category])
+
+    class Meta:
+        unique_together = ['user', 'category']
+
+    def __unicode__(self):
+        return u'[%s] %s' % (self.user, self.category)

euscanwww/djeuscan/processing/__init__.py

@@ -0,0 +1,31 @@
+import logging
+
+
+class FakeLogger(object):
+    def __getattr__(self, key):
+        return lambda *x, **y: None
+
+
+def set_verbosity_level(logger, verbosity):
+    try:
+        verbosity = int(verbosity)
+    except (ValueError, TypeError):
+        return logger
+
+    levels = {
+        0: logging.DEBUG,
+        1: logging.INFO,
+        2: logging.WARNING,
+        3: logging.ERROR,
+        4: logging.CRITICAL
+    }
+
+    if verbosity < 0:
+        verbosity = 0
+
+    if verbosity > 4:
+        verbosity = 4
+
+    logger.setLevel(levels[verbosity])
+
+    return logger

euscanwww/djeuscan/processing/regen_rrds.py

@@ -0,0 +1,31 @@
+from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
+from djeuscan import charts
+
+from djeuscan.processing import FakeLogger
+
+
+def regen_rrds(logger=None):
+    """
+    Regenerates the rrd database
+    """
+
+    if logger is None:
+        logger = FakeLogger()
+
+    logger.info("Regenering RRDs for world")
+    for wlog in WorldLog.objects.all():
+        charts.rrd_update('world', wlog.datetime, wlog)
+
+    logger.info("Regenering RRDs for categories")
+    for clog in CategoryLog.objects.all():
+        charts.rrd_update('category-%s' % clog.category,
+                          clog.datetime, clog)
+
+    logger.info("Regenering RRDs for herds")
+    for hlog in HerdLog.objects.all():
+        charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
+
+    logger.info("Regenering RRDs for maintainers")
+    for mlog in MaintainerLog.objects.all():
+        charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
+                          mlog.datetime, mlog)

euscanwww/djeuscan/processing/scan_metadata.py

@@ -0,0 +1,148 @@
+from gentoolkit.query import Query
+from gentoolkit.errors import GentoolkitFatalError
+
+from django.db.transaction import commit_on_success
+from django.core.management.color import color_style
+from django.core.exceptions import ValidationError
+
+from djeuscan.models import Package, Herd, Maintainer
+from djeuscan.processing import FakeLogger
+
+
+class ScanMetadata(object):
+    def __init__(self, logger=None):
+        self.style = color_style()
+        self.logger = logger or FakeLogger()
+
+    @commit_on_success
+    def scan(self, query=None, obj=None):
+        matches = Query(query).find(
+            include_masked=True,
+            in_installed=False,
+        )
+
+        if not matches:
+            self.logger.error(
+                self.style.ERROR("Unknown package '%s'" % query)
+            )
+            return
+
+        matches = sorted(matches)
+        pkg = matches.pop()
+        if '9999' in pkg.version and len(matches):
+            pkg = matches.pop()
+
+        if not obj:
+            obj, created = Package.objects.get_or_create(
+                category=pkg.category, name=pkg.name
+            )
+        else:
+            created = False
+
+        try:
+            obj.homepage = pkg.environment("HOMEPAGE")
+            obj.description = pkg.environment("DESCRIPTION")
+        except GentoolkitFatalError, err:
+            self.logger.error(
+                self.style.ERROR(
+                    "Gentoolkit fatal error: '%s'" % str(err)
+                )
+            )
+
+        if created:
+            self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name))
+
+        if pkg.metadata:
+            herds = dict(
+                [(herd[0], herd) for herd in pkg.metadata.herds(True)]
+            )
+            maintainers = dict(
+                [(m.email, m) for m in pkg.metadata.maintainers()]
+            )
+
+            existing_herds = [h.herd for h in obj.herds.all()]
+            new_herds = set(herds.keys()).difference(existing_herds)
+            old_herds = set(existing_herds).difference(herds.keys())
+
+            existing_maintainers = [m.email for m in obj.maintainers.all()]
+            new_maintainers = set(
+                maintainers.keys()).difference(existing_maintainers
+            )
+            old_maintainers = set(
+                existing_maintainers).difference(maintainers.keys()
+            )
+
+            for herd in obj.herds.all():
+                if herd.herd in old_herds:
+                    obj.herds.remove(herd)
+
+            for herd in new_herds:
+                herd = self.store_herd(*herds[herd])
+                obj.herds.add(herd)
+
+            for maintainer in obj.maintainers.all():
+                if maintainer.email in old_maintainers:
+                    obj.maintainers.remove(maintainer)
+
+            for maintainer in new_maintainers:
+                maintainer = maintainers[maintainer]
+                try:
+                    maintainer = self.store_maintainer(
+                        maintainer.name, maintainer.email
+                    )
+                    obj.maintainers.add(maintainer)
+                except ValidationError:
+                    self.logger.error(
+                        self.style.ERROR("Bad maintainer: '%s' '%s'" % \
+                                         (maintainer.name, maintainer.email))
+                    )
+        obj.save()
+        return True
+
+    def store_herd(self, name, email):
+        if not name:
+            name = '{nil}'
+        name = name.strip("\r").strip("\n").strip("\t").strip()
+
+        herd, created = Herd.objects.get_or_create(
+            herd=name,
+            defaults={"email": email}
+        )
+
+        if created:
+            self.logger.info('+ [h] %s <%s>' % (name, email))
+
+        herd.email = email
+        herd.save()
+
+        return herd
+
+    def store_maintainer(self, name, email):
+        if not name:
+            name = email
+        if not name:
+            name = '{nil}'
+
+        maintainer, created = Maintainer.objects.get_or_create(
+            email=email,
+            defaults={"name": name}
+        )
+
+        if created:
+            self.logger.info(
+                '+ [m] %s <%s>' % (name.encode('utf-8'), email)
+            )
+        return maintainer
+
+
+def scan_metadata(packages=None, logger=None):
+    scan_handler = ScanMetadata(logger=logger)
+    if not packages:
+        packages = Package.objects.all()
+
+    for pkg in packages:
+        if isinstance(pkg, Package):
+            result = scan_handler.scan('%s/%s' % (pkg.category, pkg.name), pkg)
+        else:
+            result = scan_handler.scan(pkg)
+    return result

euscanwww/djeuscan/processing/scan_portage.py

@@ -0,0 +1,307 @@
+import subprocess
+import portage
+import os
+import re
+from xml.dom.minidom import parseString
+
+from django.db.transaction import commit_on_success
+from django.core.management.color import color_style
+
+from euscan.helpers import get_version_type
+
+from djeuscan.processing import FakeLogger
+from djeuscan.models import Package, Version, VersionLog
+
+
+class ScanPortage(object):
+    def __init__(self, logger=None, no_log=False, purge_packages=False,
+                 kill_versions=False):
+        self.logger = logger or FakeLogger()
+        self.no_log = no_log
+        self.purge_packages = purge_packages
+        self.kill_versions = kill_versions
+
+        self.style = color_style()
+        self._cache = {'packages': {}, 'versions': {}}
+        self._overlays = None
+
+    def cache_hash_package(self, category, name):
+        return '%s/%s' % (category, name)
+
+    def cache_store_package(self, package):
+        key = self.cache_hash_package(package.category, package.name)
+        self._cache['packages'][key] = package
+
+    def cache_get_package(self, category, name):
+        return self._cache['packages'].get(
+            self.cache_hash_package(category, name)
+        )
+
+    def cache_hash_version(self, category, name, version, revision, slot,
+                           overlay):
+        key = '%s/%s-%s-r%s %s %s' % (category, name,
+                                      version, revision,
+                                      slot, overlay)
+        return key
+
+    def cache_get_version(self, category, name, version, revision, slot,
+                          overlay):
+        key = self.cache_hash_version(category, name, version, revision, slot,
+                                      overlay)
+        return self._cache['versions'].get(key)
+
+    def cache_store_version(self, version):
+        key = self.cache_hash_version(
+            version.package.category, version.package.name, version.version,
+            version.revision, version.slot, version.overlay
+        )
+        self._cache['versions'][key] = version
+
+    def overlays(self):
+        if self._overlays:
+            return self._overlays
+
+        env = os.environ
+        env['OVERLAYS_LIST'] = 'all'
+        env['PRINT_COUNT_ALWAYS'] = 'never'
+
+        cmd = ['eix', '-!']
+
+        output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
+            communicate()[0]
+        output = output.strip().strip('\n').split('\n')
+
+        overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
+
+        self._overlays = {}
+
+        for line in output:
+            match = overlay_re.match(line)
+            if not match:
+                continue
+            self._overlays[match.group('key')] = match.group('name')
+
+        return self._overlays
+
+    def scan(self, query=None):
+        if self.purge_packages:
+            with commit_on_success():
+                for package in Package.objects.all():
+                    self.logger.info('- [p] %s' % (package))
+                    package.delete()
+
+        cmd = ['eix', '--xml', '--pure-packages', '-x']
+        if query:
+            cmd.extend(['--exact', query])
+
+        if self.kill_versions:
+            self.logger.info('Killing existing versions...')
+            Version.objects.filter(packaged=True).update(alive=False)
+            self.logger.info('done')
+
+        output = subprocess.Popen(cmd, stdout=subprocess.PIPE).\
+            communicate()[0]
+
+        if len(output) == 0:
+            if not query:
+                return
+            if self.purge_packages:
+                self.logger.info('- [p] %s' % (query))
+                if '/' in query:
+                    cat, pkg = portage.catsplit(query)
+                    Package.objects.filter(category=cat, name=pkg).delete()
+                else:
+                    Package.objects.filter(name=query).delete()
+            else:
+                self.logger.error(
+                    self.style.ERROR(
+                        "Unknown package '%s'" % query
+                    )
+                )
+            return
+
+        dom = parseString(output)
+
+        for category_tag in dom.getElementsByTagName("category"):
+            for package_tag in category_tag.getElementsByTagName("package"):
+                cat = category_tag.getAttribute("name")
+                pkg = package_tag.getAttribute("name")
+                homepage_tags = package_tag.getElementsByTagName("homepage")
+                try:
+                    homepage = homepage_tags[0].firstChild.nodeValue
+                except (IndexError, AttributeError):
+                    homepage = ""
+                desc_tags = package_tag.getElementsByTagName("description")
+                try:
+                    desc = desc_tags[0].firstChild.nodeValue
+                except (IndexError, AttributeError):
+                    desc = ""
+
+                with commit_on_success():
+                    package = self.store_package(cat, pkg, homepage, desc)
+
+                    for version_tag in package_tag.\
+                                       getElementsByTagName("version"):
+                        cpv = "%s/%s-%s" % (cat, pkg,
+                                            version_tag.getAttribute("id"))
+                        slot = version_tag.getAttribute("slot")
+                        overlay = version_tag.getAttribute("overlay")
+                        self.store_version(package, cpv, slot, overlay)
+
+    def store_package(self, cat, pkg, homepage, description):
+        created = False
+        obj = self.cache_get_package(cat, pkg)
+
+        if not obj:
+            obj, created = Package.objects.get_or_create(
+                category=cat,
+                name=pkg,
+                homepage=homepage,
+                description=description,
+            )
+            self.cache_store_package(obj)
+
+        if created:
+            self.logger.info('+ [p] %s/%s' % (cat, pkg))
+
+        # Set all versions dead, then set found versions alive and
+        # delete old versions
+        if not self.kill_versions:
+            Version.objects.filter(
+                package=obj,
+                packaged=True
+            ).update(alive=False)
+
+        return obj
+
+    def store_version(self, package, cpv, slot, overlay):
+        cat, pkg, ver, rev = portage.catpkgsplit(cpv)
+
+        overlays = self.overlays()
+
+        if overlay in overlays:
+            overlay = overlays[overlay]
+        else:
+            overlay = 'gentoo'
+
+        created = False
+        obj = self.cache_get_version(
+            package.category, package.name, ver, rev, slot, overlay
+        )
+        if not obj:
+            obj, created = Version.objects.get_or_create(
+                package=package, slot=slot,
+                revision=rev, version=ver,
+                overlay=overlay,
+                defaults={
+                    "alive": True,
+                    "packaged": True,
+                    "version_type": get_version_type(ver),
+                    "confidence": 100,
+                    "handler": "portage"
+                }
+            )
+        if not created:  # Created objects have defaults values
+            obj.alive = True
+            obj.packaged = True
+            obj.save()
+
+        if created:
+            self.cache_store_version(obj)
+
+        # nothing to do (note: it can't be an upstream version because
+        # overlay can't be empty here)
+        if not created:
+            return
+
+        self.logger.info('+ [v] %s' % (obj))
+
+        if overlay == 'gentoo':
+            package.n_packaged += 1
+        else:
+            package.n_overlay += 1
+        package.n_versions += 1
+        package.save()
+
+        if self.no_log:
+            return
+
+        VersionLog.objects.create(
+            package=obj.package,
+            action=VersionLog.VERSION_ADDED,
+            slot=obj.slot,
+            revision=obj.revision,
+            version=obj.version,
+            overlay=obj.overlay
+        )
+
+
+@commit_on_success
+def do_purge_versions(logger=None, no_log=False):
+    logger = logger or FakeLogger()
+
+    # For each dead versions
+    for version in Version.objects.filter(packaged=True, alive=False):
+        if version.overlay == 'gentoo':
+            version.package.n_packaged -= 1
+        else:
+            version.package.n_overlay -= 1
+        version.package.n_versions -= 1
+        version.package.save()
+
+        logger.info('- [v] %s' % (version))
+
+        if no_log:
+            continue
+
+        VersionLog.objects.create(
+            package=version.package,
+            action=VersionLog.VERSION_REMOVED,
+            slot=version.slot,
+            revision=version.revision,
+            version=version.version,
+            overlay=version.overlay
+        )
+
+    Version.objects.filter(packaged=True, alive=False).delete()
+
+
+def scan_portage(packages=None, no_log=False, purge_packages=False,
+                 purge_versions=False, prefetch=False, logger=None):
+
+    logger = logger or FakeLogger()
+    kill_versions = False
+    if packages is None:
+        prefetch = True
+        kill_versions = True
+
+    scan_handler = ScanPortage(
+        logger=logger,
+        no_log=no_log,
+        purge_packages=purge_packages,
+        kill_versions=kill_versions,
+    )
+
+    logger.info('Scanning portage tree...')
+
+    if prefetch:
+        logger.info('Prefetching objects...')
+        for package in Package.objects.all():
+            scan_handler.cache_store_package(package)
+        for version in Version.objects.select_related('package').all():
+            scan_handler.cache_store_version(version)
+        logger.info('done')
+
+    if not packages:
+        scan_handler.scan()
+    else:
+        for pkg in packages:
+            if isinstance(pkg, Package):
+                scan_handler.scan('%s/%s' % (pkg.category, pkg.name))
+            else:
+                scan_handler.scan(pkg)
+
+    if purge_versions:
+        do_purge_versions(logger=logger, no_log=no_log)
+    logger.info('Done.')
+    return True

euscanwww/djeuscan/processing/scan_upstream.py

@@ -0,0 +1,161 @@
+import portage
+
+from django.utils import timezone
+from django.db.transaction import commit_on_success
+
+from euscan import CONFIG, output
+from euscan.scan import scan_upstream as euscan_scan_upstream
+
+from djeuscan.processing import FakeLogger
+from djeuscan.models import Package, Version, EuscanResult, VersionLog
+
+
+class ScanUpstream(object):
+    def __init__(self, logger=None):
+        self.logger = logger or FakeLogger()
+
+    def scan(self, package):
+        CONFIG["format"] = "dict"
+        output.clean()
+        output.set_query(package)
+
+        euscan_scan_upstream(package)
+
+        out = output.get_formatted_output()
+        out_json = output.get_formatted_output("json")
+
+        try:
+            cpv = out[package]["metadata"]["cpv"]
+            scan_time = out[package]["metadata"]["scan_time"]
+            ebuild = out[package]["metadata"]["ebuild"]
+        except KeyError:
+            return {}
+
+        with commit_on_success():
+            obj = self.store_package(cpv)
+
+            for res in out[package]["result"]:
+                self.store_version(
+                    obj,
+                    res["version"],
+                    " ".join(res["urls"]),
+                    res["type"],
+                    res["handler"],
+                    res["confidence"],
+                )
+
+            self.store_result(obj, out_json, scan_time, ebuild)
+
+        return out
+
+    def store_result(self, package, formatted_log, scan_time, ebuild):
+        # Remove previous logs
+        EuscanResult.objects.filter(package=package).delete()
+
+        obj = EuscanResult()
+        obj.package = package
+        obj.result = formatted_log
+        obj.datetime = timezone.now()
+        obj.scan_time = scan_time
+        obj.ebuild = ebuild
+        obj.save()
+
+    def store_package(self, cpv):
+        cat, pkg, ver, rev = portage.catpkgsplit(cpv)
+
+        obj, created = Package.objects.get_or_create(category=cat, name=pkg)
+
+        if created:
+            self.logger.info('+ [p] %s/%s' % (cat, pkg))
+
+        # Set all versions dead, then set found versions alive and
+        # delete old versions
+        Version.objects.filter(package=obj, packaged=False).update(alive=False)
+
+        return obj
+
+    def store_version(self, package, ver, url, version_type, handler,
+                      confidence):
+        obj, created = Version.objects.get_or_create(
+            package=package,
+            slot='',
+            revision='r0',
+            version=ver,
+            overlay='',
+            defaults={"alive": True, "urls": url, "packaged": False,
+                      "version_type": version_type, "handler": handler,
+                      "confidence": confidence}
+        )
+        if not created:
+            obj.alive = True
+            obj.urls = url
+            obj.packaged = False
+            obj.save()
+
+        # If it's not a new version, just update the object and continue
+        if not created:
+            return
+
+        self.logger.info('+ [u] %s %s' % (obj, url))
+
+        VersionLog.objects.create(
+            package=package,
+            action=VersionLog.VERSION_ADDED,
+            slot='',
+            revision='r0',
+            version=ver,
+            overlay=''
+        )
+
+        package.n_versions += 1
+        package.save()
+
+
+@commit_on_success
+def do_purge_versions(logger=None):
+    logger = logger or FakeLogger()
+
+    # For each dead versions
+    for version in Version.objects.filter(packaged=False, alive=False):
+        VersionLog.objects.create(
+            package=version.package,
+            action=VersionLog.VERSION_REMOVED,
+            slot=version.slot,
+            revision=version.revision,
+            version=version.version,
+            overlay=version.overlay
+        )
+
+        version.package.n_versions -= 1
+        version.package.save()
+
+        logger.info('- [u] %s %s' % (version, version.urls))
+    Version.objects.filter(packaged=False, alive=False).delete()
+
+
+def scan_upstream(packages=None, purge_versions=False,
+                  logger=None):
+    logger = logger or FakeLogger()
+
+    scan_handler = ScanUpstream(logger=logger)
+
+    logger.info('Scanning upstream...')
+
+    if not packages:
+        packages = Package.objects.all()
+
+    result = True
+
+    for pkg in packages:
+        if isinstance(pkg, Package):
+            curr = scan_handler.scan('%s/%s' % (pkg.category, pkg.name))
+        else:
+            curr = scan_handler.scan(pkg)
+        if not curr:
+            result = False
+
+    if purge_versions:
+        do_purge_versions(logger=logger)
+
+    logger.info('Done.')
+    return result

euscanwww/djeuscan/processing/update_counters.py

@@ -0,0 +1,175 @@
+from django.db.transaction import commit_on_success
+from django.utils import timezone
+
+from djeuscan.models import Package, Herd, Maintainer, Version
+from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
+from djeuscan import charts
+from djeuscan.processing import FakeLogger
+
+from distutils.version import StrictVersion, LooseVersion
+
+
+def _compare_versions(version1, version2):
+    try:
+        return cmp(StrictVersion(version1), StrictVersion(version2))
+    # in case of abnormal version number, fall back to LooseVersion
+    except ValueError:
+        return cmp(LooseVersion(version1), LooseVersion(version2))
+
+
+def _add_safe(storage, key):
+    if key not in storage:
+        storage[key] = 1
+    else:
+        storage[key] += 1
+
+
+def _add_last_ver(storage, version):
+    key = version['package_id']
+    if key not in storage:
+        storage[key] = version
+        return
+    if version['version'].startswith('9999'):
+        return
+    if _compare_versions(storage[key]['version'],
+                        version['version']) < 0:
+        storage[key] = version
+
+
+@commit_on_success
+def update_counters(fast=False, nolog=False, logger=None):
+    logger = logger or FakeLogger()
+
+    now = timezone.now()
+
+    categories = {}
+    herds = {}
+    maintainers = {}
+
+    wlog = None
+
+    if not nolog:
+        wlog = WorldLog()
+        wlog.datetime = now
+
+        for cat in Package.objects.values('category').distinct():
+            clog = CategoryLog()
+            clog.datetime = now
+            clog.category = cat['category']
+            categories[clog.category] = clog
+
+        for herd in Herd.objects.all():
+            hlog = HerdLog()
+            hlog.datetime = now
+            hlog.herd = herd
+            herds[herd.id] = hlog
+
+        for maintainer in Maintainer.objects.all():
+            mlog = MaintainerLog()
+            mlog.datetime = now
+            mlog.maintainer = maintainer
+            maintainers[maintainer.id] = mlog
+
+    package_queryset = Package.objects.all()
+
+    n_versions = {}
+    n_packaged = {}
+    n_overlay = {}
+
+    last_versions_gentoo = {}
+    last_versions_overlay = {}
+    last_versions_upstream = {}
+
+    if not fast:
+        attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
+        for version in Version.objects.all().values(*attrs):
+            overlay, packaged = version['overlay'], version['packaged']
+            package_id = version['package_id']
+
+            _add_safe(n_versions, package_id)
+
+            if not packaged:
+                _add_last_ver(last_versions_upstream, version)
+                continue
+            if overlay == 'gentoo':
+                _add_safe(n_packaged, package_id)
+                _add_last_ver(last_versions_gentoo, version)
+            else:
+                _add_safe(n_overlay, package_id)
+                _add_last_ver(last_versions_overlay, version)
+
+    for package in package_queryset.select_related('herds', 'maintainers'):
+        if not fast:
+            package.n_versions = n_versions.get(package.id, 0)
+            package.n_packaged = n_packaged.get(package.id, 0)
+            package.n_overlay = n_overlay.get(package.id, 0)
+
+            default = {'id': None}
+            package.last_version_gentoo_id = last_versions_gentoo.get(
+                package.id, default
+            )['id']
+            package.last_version_overlay_id = last_versions_overlay.get(
+                package.id, default
+            )['id']
+            package.last_version_upstream_id = last_versions_upstream.get(
+                package.id, default
+            )['id']
+
+            package.save()
+
+        n_packages_gentoo = int(package.n_packaged == package.n_versions)
+        n_packages_overlay = int(package.n_overlay and package.n_packaged \
+                                 + package.n_overlay == package.n_versions)
+        n_packages_outdated = int(package.n_packaged + package.n_overlay \
+                                  < package.n_versions)
+
+        def update_row(storage, key):
+            storage[key].n_packages_gentoo += n_packages_gentoo
+            storage[key].n_packages_overlay += n_packages_overlay
+            storage[key].n_packages_outdated += n_packages_outdated
+
+            storage[key].n_versions_gentoo += package.n_packaged
+            storage[key].n_versions_overlay += package.n_overlay
+            storage[key].n_versions_upstream += package.n_versions - \
+                                                package.n_packaged - \
+                                                package.n_overlay
+
+        def update_log(storage, qs):
+            for row in qs:
+                update_row(storage, row['id'])
+
+        if not nolog:
+            update_log(herds, package.herds.all().values('id'))
+            update_log(maintainers, package.maintainers.all().values('id'))
+            update_row(categories, package.category)
+
+            wlog.n_packages_gentoo += n_packages_gentoo
+            wlog.n_packages_overlay += n_packages_overlay
+            wlog.n_packages_outdated += n_packages_outdated
+
+            wlog.n_versions_gentoo += package.n_packaged
+            wlog.n_versions_overlay += package.n_overlay
+            wlog.n_versions_upstream += package.n_versions - \
+                                        package.n_packaged - \
+                                        package.n_overlay
+
+    if nolog:
+        return
+
+    for clog in categories.values():
+        logger.info('+ [cl] %s\n' % clog)
+        charts.rrd_update('category-%s' % clog.category, now, clog)
+        clog.save()
+
+    for hlog in herds.values():
+        logger.info('+ [hl] %s\n' % hlog)
+        charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
+        hlog.save()
+
+    for mlog in maintainers.values():
+        logger.info('+ [ml] %s\n' % mlog)
+        charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
+        mlog.save()
+
+    charts.rrd_update('world', now, wlog)
+    wlog.save()

euscanwww/djeuscan/processing/update_portage_trees.py

@@ -0,0 +1,66 @@
+import subprocess
+from StringIO import StringIO
+
+from django.conf import settings
+
+from djeuscan.processing import FakeLogger
+
+
+def _launch_command(cmd):
+    """
+    Helper for launching shell commands inside tasks
+    """
+    fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+    output = StringIO(fp.communicate()[0])
+    return output.getvalue()
+
+
+def emerge_sync():
+    """
+    Launches an emerge --sync
+    """
+    cmd = ["emerge", "--sync", "--root", settings.PORTAGE_ROOT,
+           "--config-root", settings.PORTAGE_CONFIGROOT]
+    return _launch_command(cmd)
+
+
+def layman_sync():
+    """
+    Syncs Layman repos
+    """
+    from layman import Layman
+    l = Layman(config=settings.LAYMAN_CONFIG)
+    return l.sync(l.get_installed(), output_results=False)
+
+
+def emerge_regen():
+    """
+    Launches emerge --regen
+    """
+    cmd = [
+        "emerge", "--regen", "--jobs", settings.EMERGE_REGEN_JOBS, "--root",
+        settings.PORTAGE_ROOT, "--config-root", settings.PORTAGE_CONFIGROOT
+    ]
+    return _launch_command(cmd)
+
+
+def eix_update():
+    """
+    Launches eix-update
+    """
+    cmd = ["eix-update"]
+    return _launch_command(cmd)
+
+
+def update_portage_trees(logger=None):
+    logger = logger or FakeLogger()
+    logger.info("Running emerge --sync")
+    emerge_sync()
+    logger.info("Running layman --sync")
+    layman_sync()
+    logger.info("Running emerge --regen")
+    emerge_regen()
+    logger.info("Running eix-update")
+    eix_update()
+    logger.info("Done!")

euscanwww/djeuscan/tasks.py

@@ -0,0 +1,292 @@
+"""
+Celery tasks for djeuscan
+"""
+
+from itertools import islice
+
+from celery.task import task, periodic_task
+from celery.task.schedules import crontab
+from celery.task.sets import TaskSet
+
+from django.conf import settings
+
+from djeuscan.models import Package, RefreshPackageQuery
+from djeuscan.processing.regen_rrds import regen_rrds
+from djeuscan.processing.update_counters import update_counters
+from djeuscan.processing.scan_metadata import scan_metadata
+from djeuscan.processing.scan_portage import scan_portage
+from djeuscan.processing.scan_upstream import scan_upstream
+from djeuscan.processing.update_portage_trees import update_portage_trees
+
+
+class TaskFailedException(Exception):
+    """
+    Exception for failed tasks
+    """
+    pass
+
+
+def _chunks(it, n):
+    """
+    Chunk generator, takes an iterator and the desired size of the chunk
+    """
+    for first in it:
+        yield [first] + list(islice(it, n - 1))
+
+
+def _run_in_chunks(task, packages, kwargs=None,
+                   concurrently=settings.TASKS_CONCURRENTLY,
+                   n=settings.TASKS_SUBTASK_PACKAGES):
+    """
+    Launches a TaskSet at a time with <concurrently> subtasks.
+    Each subtask has <n> packages to handle
+    """
+    output = []
+
+    chunk_generator = _chunks(iter(packages), n)
+    done = False
+
+    while not done:
+        tasks = []
+        for _ in range(concurrently):
+            try:
+                chunk = chunk_generator.next()
+            except StopIteration:
+                done = True
+            else:
+                tasks.append(task.subtask((chunk, ), kwargs))
+        job = TaskSet(tasks=tasks)
+        result = job.apply_async()
+        # TODO: understand why this causes timeout
+        output.extend(list(result.join()))
+    return output
+
+
+@task
+def regen_rrds_task():
+    """
+    Regenerate RRDs
+    """
+    return regen_rrds()
+
+
+@task
+def update_counters_task(fast=True):
+    """
+    Updates counters
+    """
+    return update_counters(fast=fast)
+
+
+@task
+def _scan_metadata_task(packages):
+    """
+    Scans metadata for the given set of packages
+    """
+    logger = _scan_metadata_task.get_logger()
+    logger.info("Starting metadata scanning subtask for %d packages...",
+                len(packages))
+
+    result = scan_metadata(
+        packages=packages,
+        logger=logger,
+    )
+    if not result:
+        raise TaskFailedException
+    return result
+
+
+@task
+def scan_metadata_list_task(query):
+    """
+    Runs a parallel metadata scan for packages in the query list (space
+    separated string). Task used only from the web interface.
+    """
+    return _run_in_chunks(_scan_metadata_task, [p for p in query.split()])
+
+
+@task
+def scan_metadata_all_task():
+    """
+    Runs a parallel metadata scan for all packages
+    """
+    return _run_in_chunks(_scan_metadata_task, Package.objects.all())
+
+
+@task
+def _scan_portage_task(packages, no_log=False, purge_packages=False,
+                       purge_versions=False, prefetch=False):
+    """
+    Scans portage for the given set of packages
+    """
+    logger = _scan_portage_task.get_logger()
+    if packages:
+        logger.info("Starting portage scanning subtask for %d packages...",
+                    len(packages))
+    else:
+        logger.info("Starting portage scanning for all packages...")
+
+    result = scan_portage(
+        packages=packages,
+        no_log=no_log,
+        purge_packages=purge_packages,
+        purge_versions=purge_versions,
+        prefetch=prefetch,
+        logger=logger,
+    )
+    if not result:
+        raise TaskFailedException
+    return result
+
+
+@task
+def scan_portage_list_task(query, no_log=False, purge_packages=False,
+                           purge_versions=False, prefetch=False):
+    """
+    Runs a parallel portage scan for packages in the query list (space
+    separated string). Task used only from the web interface.
+    """
+    kwargs = {"no_log": no_log, "purge_packages": purge_packages,
+              "purge_versions": purge_versions, "prefetch": prefetch}
+    return _run_in_chunks(_scan_portage_task, [p for p in query.split()],
+                          kwargs)
+
+
+@task
+def scan_portage_all_task(no_log=False, purge_packages=False,
+                          purge_versions=False, prefetch=False):
+    """
+    Runs a syncronous portage scan for all packages
+    """
+    return _scan_portage_task(
+        packages=None,
+        no_log=no_log,
+        purge_packages=purge_packages,
+        purge_versions=purge_versions,
+        prefetch=prefetch,
+    )
+
+
+@task
+def _scan_upstream_task(packages, purge_versions=False):
+    """
+    Scans upstream for the given set of packages
+    """
+    logger = _scan_upstream_task.get_logger()
+
+    logger.info("Starting upstream scanning subtask for %d packages...",
+                len(packages))
+
+    result = scan_upstream(
+        packages=packages,
+        purge_versions=purge_versions,
+        logger=logger,
+    )
+    if not result:
+        raise TaskFailedException
+    return result
+
+
+@task
+def scan_upstream_list_task(query, purge_versions=False):
+    """
+    Runs a parallel upstream scan for packages in the query list (space
+    separated string). Task used only from the web interface.
+    """
+
+    kwargs = {"purge_versions": purge_versions}
+    return _run_in_chunks(_scan_upstream_task, [p for p in query.split()],
+                          kwargs)
+
+
+@task
+def scan_upstream_all_task(purge_versions=False):
+    """
+    Runs a parallel portage scan for all packages
+    """
+    kwargs = {"purge_versions": purge_versions}
+    return _run_in_chunks(_scan_upstream_task, Package.objects.all(), kwargs)
+
+
+@task
+def update_portage_trees_task():
+    """
+    Update portage tree
+    """
+    logger = update_portage_trees_task.get_logger()
+    update_portage_trees(logger=logger)
+
+
+@task
+def update_task(update_portage_trees=True, scan_portage=True,
+                scan_metadata=True, scan_upstream=True, update_counter=True):
+    """
+    Update the whole euscan system
+    """
+    if update_portage_trees:
+        update_portage_trees_task()
+    if scan_portage:
+        scan_portage_all_task(prefetch=True, purge_packages=True,
+                              purge_versions=True)
+
+    # metadata and upstream scan can run concurrently, launch them
+    # asynchronously and wait for them to finish
+    metadata_job = None
+    if scan_metadata:
+        metadata_job = scan_metadata_all_task().delay()
+
+    upstream_job = None
+    if scan_upstream:
+        upstream_job = scan_upstream_all_task().delay()
+
+    if metadata_job:
+        metadata_job.wait()
+    if upstream_job:
+        upstream_job.wait()
+
+    update_counters(fast=False)
+
+
+@task
+def scan_package_task(package):
+    _scan_portage_task([package], purge_packages=True, purge_versions=True)
+    _scan_metadata_task([package])
+    _scan_upstream_task([package])
+
+
+@periodic_task(run_every=crontab(minute="*/1"))
+def consume_refresh_package_request():
+    """
+    Satisfies user requests for package refreshing, runs every minute
+    """
+    try:
+        obj = RefreshPackageQuery.objects.all().order_by('-priority')[0]
+    except IndexError:
+        return {}
+    else:
+        result = scan_package_task(obj.query)
+        obj.delete()
+        return result
+
+
+@periodic_task(run_every=crontab(hour=03, minute=00, day_of_week=1))
+def update_periodic_task():
+    """
+    Runs a whole update once a week
+    """
+    update_task()
+
+
+admin_tasks = [
+    regen_rrds_task,
+    update_counters_task,
+    scan_metadata_list_task,
+    scan_metadata_all_task,
+    scan_portage_all_task,
+    scan_portage_list_task,
+    scan_upstream_all_task,
+    scan_upstream_list_task,
+    update_portage_trees_task,
+    update_task,
+    scan_package_task,
+]

euscanwww/djeuscan/templates/_base.html

@@ -10,6 +10,7 @@
     <link rel="alternate" type="application/atom+xml" title="Global log" href="{% url "global_feed" %}" />
     {% endblock %}
     {% block css %}
+    <link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/bootstrap.min.css" media="screen" title="Normal" />
     <link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/style.css" media="screen" title="Normal" />
     {% endblock %}
     {% block javascript %}
@@ -22,12 +23,20 @@
       </a>
       {% block header %}<h1>Ebuild Upstream Scanner (euscan)</h1>{% endblock %}
     </div>
-    <div id="content">
+    <div class="row-fluid">
+      <div class="span10">
+        <div class="row-fluid">
+          <div class="span1"></div>
+          <div id="content" class="rounded span11">
             {% block content %}{% endblock %}
           </div>
-    <div id="menus">
+        </div>
+      </div>
+      <div class="span2">
+        <div class="row-fluid">
+          <div id="menus" class="span11">
             {% block menus %}
-        <div class="menu">
+              <div class="menu rounded">
                 <ul>
                   {% block menu %}
                     <li><a href="{% url "index" %}">Home</a></li>
@@ -37,12 +46,25 @@
                     <li><a href="{% url "overlays" %}">Overlays</a></li>
                     <li><a href="{% url "world" %}">Scan World</a></li>
                     <li><a href="{% url "statistics" %}">Statistics</a></li>
-              <!--
+
                     <li>---</li>
-              <li><a href="#">Login</a></li>
+
-              <li><a href="#">Register</a></li>
+                    {% if user.is_authenticated %}
-              -->
+                      <li><a href="{% url "accounts_index" %}">{{ user }}'s profile</a></li>
+                      <ul class="submenu">
+                        <li><a href="{% url "accounts_categories" %}">Categories</a></li>
+                        <li><a href="{% url "accounts_herds" %}">Herds</a></li>
+                        <li><a href="{% url "accounts_maintainers" %}">Maintainers</a></li>
+                        <li><a href="{% url "accounts_packages" %}">Packages</a></li>
+                      </ul>
+                      <li><a href="{% url "django.contrib.auth.views.logout" %}">Logout</a></li>
+                    {% else %}
+                      <li><a href="{% url "django.contrib.auth.views.login" %}?next={% url "accounts_index" %}">Login</a></li>
+                      <li><a href="{% url "registration.views.register" %}">Register</a></li>
+                    {% endif %}
+
                     <li>---</li>
+
                     {% block menu_feed %}
                       <li>
                         <img src="{{ STATIC_URL }}img/feed.png" alt="feed" />
@@ -57,6 +79,10 @@
               </div>
             {% endblock %}
           </div>
+          <div class="span1"></div>
+        </div>
+      </div>
+    </div>
     <div id="footer">
       <p>
       {% block last_scan %}

euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html

@@ -0,0 +1,81 @@
+{% extends "admin/change_list.html" %}
+
+{% load url from future %}
+
+{% block object-tools %}
+  <div>
+    <span id="task_selector"></span>
+    <form id="task_data"></form>
+    <a href="#" id="task_launch">Launch task</a>
+  </div>
+
+  <script type="text/javascript">
+    (function($){
+      $(document).ready(function() {
+
+        $.get("{% url "registered_tasks" %}", function (data) {
+          var selector = $("<select/>");
+          selector.append(
+            $("<option/>").text("-------------").attr("selected", "selected")
+          );
+          $.each(data.tasks, function(task) {
+            var t = data["tasks"][task];
+            selector.append(
+              $("<option/>").val(task).text(task)
+            )
+          });
+
+          selector.change(function() {
+            var task = data["tasks"][$(this).val()];
+            $("#task_data").empty();
+
+            if (task.args) {
+              $.each(task.args, function(i) {
+                var arg = task.args[i]
+                  , arg_input = $("<input/>").attr("name", arg)
+                  ,  default_arg;
+
+                if (task.defaults) {
+                  var default_arg_i = task.args.length - 1 - i;
+                  if (task.defaults[default_arg_i] !== undefined) {
+                    default_arg = task.defaults[default_arg_i];
+                    arg_input.val(default_arg);
+
+                    if (task.defaults_types && task.defaults_types[default_arg_i] === "bool") {
+                      arg_input.attr("type", "checkbox").attr("checked", default_arg).val("true");
+                    }
+                    else {
+                      return true;
+                    }
+                  }
+                }
+
+                $("#task_data").append(
+                  $("<label/>").text(arg + ": ").attr("for", arg)
+                ).append(
+                  arg_input
+                );
+              });
+            }
+          })
+
+          $("#task_selector").append(selector);
+        });
+
+        $("#task_launch").click(function() {
+            var task_name = $("#task_selector").find("select").val();
+            var data = $("#task_data").serialize();
+            var url = "{% url "apply_task" "task_name" %}";
+
+            $.post(url.replace("task_name", task_name), data, function() {
+              alert("Submitted!");
+              location.reload();
+            });
+        });
+
+        setTimeout(location.reload, 30000);
+
+      });
+    })(django.jQuery);
+  </script>
+{% endblock %}

euscanwww/djeuscan/templates/euscan/_categories_table.html

@@ -0,0 +1,32 @@
+{% load url from future %}
+{% load euscan %}
+
+<table id="table" class="display">
+  <thead>
+    <th>Category</th>
+    <th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
+    <th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
+    <th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
+    <th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
+    {% if extras %}
+    <th>Graphs</th>
+    {% endif %}
+  </thead>
+  <tbody>
+  {% for category in categories %}
+  <tr>
+    <td>
+      <a href="{% url "category" category.category %}">{{ category.category }}</a>
+      {% package_bar category %}
+    </td>
+    {% package_cols category %}
+    {% if extras %}
+    <td>
+      <img src="{% url "chart_category" category.category 'packages-monthly-small' %}" />
+      <img src="{% url "chart_category" category.category 'versions-monthly-small' %}" />
+    </td>
+    {% endif %}
+  </tr>
+  {% endfor %}
+  </tbody>
+</table>

euscanwww/djeuscan/templates/euscan/_datatable.html

@@ -24,7 +24,7 @@
 
 {% block menus %}
 {{ block.super }}
-<div class="menu">
+<div class="menu notfirst-menu rounded">
   <ul>
     <li><img src="{{ STATIC_URL }}img/gentoo-icon.png" /> Gentoo</li>
     <li><img src="{{ STATIC_URL }}img/overlay-icon.png" /> Overlays</li>

euscanwww/djeuscan/templates/euscan/_herds_table.html

@@ -0,0 +1,34 @@
+{% load url from future %}
+{% load euscan %}
+
+<table id="table" class="display">
+  <thead>
+    <th>Herd</th>
+    <th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
+    <th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
+    <th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
+    <th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
+    {% if extras %}
+    <th>Graphs</th>
+    {% endif %}
+  </thead>
+  <tbody>
+  {% for herd in herds %}
+    <tr>
+      <td>
+        <a href="{% url "herd" herd.herds__herd %}">
+          {{ herd.herds__herd }}
+        </a>
+        {% package_bar herd %}
+      </td>
+      {% package_cols herd %}
+      {% if extras %}
+        <td>
+          <img src="{% url "chart_herd" herd.herds__herd 'packages-monthly-small' %}" />
+          <img src="{% url "chart_herd" herd.herds__herd 'versions-monthly-small' %}" />
+        </td>
+      {% endif %}
+    </tr>
+  {% endfor %}
+  </tbody>
+</table>

euscanwww/djeuscan/templates/euscan/_maintainers_table.html

@@ -0,0 +1,40 @@
+{% load url from future %}
+{% load euscan %}
+
+<table id="table" class="display">
+  <thead>
+    <th>Maintainer</th>
+    <th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
+    <th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
+    <th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
+    <th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
+    {% if extras %}
+      <th>Graphs</th>
+    {% endif %}
+  </thead>
+  <tbody>
+  {% for maintainer in maintainers %}
+    <tr>
+      <td>
+        <a href="{% url "maintainer" maintainer.maintainers__id %}">
+          {% if maintainer.maintainers__name != maintainer.maintainers__email %}
+            {{ maintainer.maintainers__name }} &lt;{{ maintainer.maintainers__email }}&gt;
+          {% else %}
+            {{ maintainer.maintainers__name }}
+          {% endif %}
+        </a>
+        {% package_bar maintainer %}
+      </td>
+      {% package_cols maintainer %}
+
+      {% if extras %}
+        <td>
+          <img src="{% url "chart_maintainer" maintainer.maintainers__id 'packages-monthly-small' %}" />
+          <img src="{% url "chart_maintainer" maintainer.maintainers__id 'versions-monthly-small' %}" />
+        </td>
+      {% endif %}
+
+    </tr>
+  {% endfor %}
+  </tbody>
+</table>

euscanwww/djeuscan/templates/euscan/_package_cols.html

@@ -1,4 +1,4 @@
-{% load packages %}
+{% load euscan %}
 {% load mul %}
 {% load sub %}
 {% load div %}

euscanwww/djeuscan/templates/euscan/_packages.html

@@ -1,4 +1,4 @@
-{% load packages %}
+{% load euscan %}
 {% load sub %}
 {% load div %}
 {% load mul %}

euscanwww/djeuscan/templates/euscan/accounts/categories.html

@@ -0,0 +1,15 @@
+{% extends "euscan/_datatable.html" %}
+{% load url from future %}
+{% load euscan %}
+
+{% block title %}
+  {{ block.super }} - Watched categories
+{% endblock %}
+
+
+{% block content %}
+  <h2>Watched categories</h2>
+
+  {% categories_table categories %}
+
+{% endblock %}

euscanwww/djeuscan/templates/euscan/accounts/herds.html

@@ -0,0 +1,14 @@
+{% extends "euscan/_datatable.html" %}
+{% load url from future %}
+{% load euscan %}
+
+{% block title %}
+  {{ block.super }} - Watched herds
+{% endblock %}
+
+
+{% block content %}
+  <h2>Watched herds</h2>
+
+  {% herds_table herds %}
+{% endblock %}

euscanwww/djeuscan/templates/euscan/accounts/index.html

@@ -0,0 +1,11 @@
+{% extends "_base.html" %}
+{% load url from future %}
+
+{% block title %}
+  {{ block.super }} - Welcome {{ user }}
+{% endblock %}
+
+
+{% block content %}
+  <h2>Welcome {{ user }}</h2>
+{% endblock %}

euscanwww/djeuscan/templates/euscan/accounts/maintainers.html

@@ -0,0 +1,14 @@
+{% extends "euscan/_datatable.html" %}
+{% load url from future %}
+{% load euscan %}
+
+{% block title %}
+  {{ block.super }} - Watched maintainers
+{% endblock %}
+
+
+{% block content %}
+  <h2>Watched maintainers</h2>
+
+  {% maintainers_table maintainers %}
+{% endblock %}

euscanwww/djeuscan/templates/euscan/accounts/packages.html

@@ -0,0 +1,16 @@
+{% extends "euscan/_datatable.html" %}
+{% load url from future %}
+
+{% load euscan %}
+
+{% block title %}
+  {{ block.super }} - Watched packages
+{% endblock %}
+
+
+{% block content %}
+  <h2>Watched packages</h2>
+
+  {% packages packages %}
+
+{% endblock %}

euscanwww/djeuscan/templates/euscan/categories.html

@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 {% load url from future %}
 
 {% block title %}
@@ -10,33 +10,6 @@
 {% block content %}
 <h2>Categories</h2>
 
-<table id="table" class="display">
+{% categories_table categories request.GET.extras %}
-  <thead>
+
-    <th>Category</th>
-    <th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
-    <th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
-    <th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
-    <th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
-    {% if request.GET.extras %}
-    <th>Graphs</th>
-    {% endif %}
-  </thead>
-  <tbody>
-  {% for category in categories %}
-  <tr>
-    <td>
-      <a href="{% url "category" category.category %}">{{ category.category }}</a>
-      {% package_bar category %}
-    </td>
-    {% package_cols category %}
-    {% if request.GET.extras %}
-    <td>
-      <img src="{% url "chart_category" category.category 'packages-monthly-small' %}" />
-      <img src="{% url "chart_category" category.category 'versions-monthly-small' %}" />
-    </td>
-    {% endif %}
-  </tr>
-  {% endfor %}
-  </tbody>
-</table>
 {% endblock %}

euscanwww/djeuscan/templates/euscan/category.html

@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 {% load url from future %}
 
 {% block title %}

euscanwww/djeuscan/templates/euscan/herd.html

@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 {% load url from future %}
 
 {% block title %}

euscanwww/djeuscan/templates/euscan/herds.html

@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 {% load url from future %}
 
 {% block title %}
@@ -9,35 +9,7 @@
 
 {% block content %}
 <h2>Herds</h2>
-<table id="table" class="display">
+
-  <thead>
+{% herds_table herds request.GET.extras %}
-    <th>Herd</th>
+
-    <th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
-    <th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
-    <th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
-    <th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
-    {% if request.GET.extras %}
-    <th>Graphs</th>
-    {% endif %}
-  </thead>
-  <tbody>
-  {% for herd in herds %}
-    <tr>
-      <td>
-        <a href="{% url "herd" herd.herds__herd %}">
-          {{ herd.herds__herd }}
-        </a>
-        {% package_bar herd %}
-      </td>
-      {% package_cols herd %}
-      {% if request.GET.extras %}
-        <td>
-          <img src="{% url "chart_herd" herd.herds__herd 'packages-monthly-small' %}" />
-          <img src="{% url "chart_herd" herd.herds__herd 'versions-monthly-small' %}" />
-        </td>
-      {% endif %}
-    </tr>
-  {% endfor %}
-  </tbody>
-</table>
 {% endblock %}

euscanwww/djeuscan/templates/euscan/maintainer.html

@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 {% load url from future %}
 
 {% block title %}

euscanwww/djeuscan/templates/euscan/maintainers.html

@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 {% load url from future %}
 
 {% block title %}
@@ -9,41 +9,7 @@
 
 {% block content %}
 <h2>Maintainers</h2>
-<table id="table" class="display">
-  <thead>
-    <th>Maintainer</th>
-    <th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
-    <th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
-    <th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
-    <th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
-    {% if request.GET.extras %}
-      <th>Graphs</th>
-    {% endif %}
-  </thead>
-  <tbody>
-  {% for maintainer in maintainers %}
-    <tr>
-      <td>
-        <a href="{% url "maintainer" maintainer.maintainers__id %}">
-          {% if maintainer.maintainers__name != maintainer.maintainers__email %}
-            {{ maintainer.maintainers__name }} &lt;{{ maintainer.maintainers__email }}&gt;
-          {% else %}
-            {{ maintainer.maintainers__name }}
-          {% endif %}
-        </a>
-        {% package_bar maintainer %}
-      </td>
-      {% package_cols maintainer %}
 
-      {% if request.GET.extras %}
+{% maintainers_table maintainers request.GET.extras %}
-        <td>
-          <img src="{% url "chart_maintainer" maintainer.maintainers__id 'packages-monthly-small' %}" />
-          <img src="{% url "chart_maintainer" maintainer.maintainers__id 'versions-monthly-small' %}" />
-        </td>
-      {% endif %}
 
-    </tr>
-  {% endfor %}
-  </tbody>
-</table>
 {% endblock %}

euscanwww/djeuscan/templates/euscan/overlay.html

@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 
 {% block title %}
 {{ block.super }} - Overlay: {{ overlay }}

euscanwww/djeuscan/templates/euscan/overlays.html

@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 {% load url from future %}
 
 {% block title %}

euscanwww/djeuscan/templates/euscan/package.html

@@ -24,7 +24,21 @@
 {% endblock %}
 
 {% block content %}
-<h2>{{ package.category }}/{{ package.name }}</h2>
+<h2>
+  {{ package.category }}/{{ package.name }}
+
+  {% if user.is_authenticated %}
+    <button class="btn refresh-button" data-package="{{ package.category }}/{{ package.name }}">
+      Refresh
+    </button>
+    <button class="btn favourite-button {% if favourited %}hide{% endif %}" data-url="{% url "favourite_package" package.category package.name %}">
+      Add to favourites
+    </button>
+    <button class="btn unfavourite-button {% if not favourited %}hide{% endif %}" data-url="{% url "unfavourite_package" package.category package.name %}">
+      Remove from favourites
+    </button>
+  {% endif %}
+</h2>
 <dl>
   {% if package.description %}
   <dt>Description</dt>
@@ -129,4 +143,28 @@
   </dd>
   {% endif %}
 </dl>
+
+<script type="text/javascript">
+  $(".refresh-button").click(function() {
+    var url = "{% url "refresh_package" "x/x" %}";
+    $.post(url.replace("x/x", $(this).data("package")), function() {
+      alert("Submitted!");
+    });
+  });
+
+  $(".favourite-button").click(function() {
+    $.post($(this).data("url"), function() {
+      $(".unfavourite-button").removeClass("hide");
+      $(".favourite-button").addClass("hide");
+    });
+  });
+
+  $(".unfavourite-button").click(function() {
+    $.post($(this).data("url"), function() {
+      $(".favourite-button").removeClass("hide");
+      $(".unfavourite-button").addClass("hide");
+    });
+  });
+</script>
+
 {% endblock %}

euscanwww/djeuscan/templates/euscan/statistics.html

@@ -3,7 +3,6 @@
 {% load url from future %}
 
 {% block content %}
-<h2>What's euscan ?</h2>
 <h2>Statistics</h2>
 
 <h3>Current statistics</h3>

euscanwww/djeuscan/templates/euscan/world_scan.html

@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 
 {% block title %}
 {{ block.super }} - World Scan

euscanwww/djeuscan/templates/registration/_registration_base.html

@@ -0,0 +1 @@
+{% extends "_base.html" %}

euscanwww/djeuscan/templates/registration/activate.html

@@ -0,0 +1,13 @@
+{% extends "registration/_registration_base.html" %}
+
+{% block title %}{% if account %}Activation complete{% else %}Activation problem{% endif %}{% endblock %}
+
+{% block content %}
+  {% if account %}
+    Thanks {{ account }}, activation complete! You may now <a href='{% url auth_login %}'>login</a>
+    using the username and password you set at registration.
+  {% else %}
+    Oops &ndash; it seems that your activation key is invalid.
+    Please check the url again.
+  {% endif %}
+{% endblock %}

euscanwww/djeuscan/templates/registration/activation_complete.html

@@ -0,0 +1,6 @@
+{% extends "registration/_registration_base.html" %}
+{% block title %}Activation complete{% endblock %}
+{% block content %}
+  Thanks, activation complete! You may now <a href='{% url auth_login %}'>login</a>
+  using the username and password you set at registration.
+{% endblock %}

euscanwww/djeuscan/templates/registration/activation_email.txt

@@ -0,0 +1,11 @@
+You (or someone pretending to be you) have asked to register an account at
+{{ site.name }}. If this wasn't you, please ignore this email
+and your address will be removed from our records.
+
+To activate this account, please click the following link within the next
+{{ expiration_days }} days:
+
+http://{{site.domain}}{% url registration_activate activation_key %}
+
+Sincerely,
+{{ site.name }} Management

euscanwww/djeuscan/templates/registration/activation_email_subject.txt

@@ -0,0 +1 @@
+Account registration for {{ site.name }}

euscanwww/djeuscan/templates/registration/login.html

@@ -0,0 +1,41 @@
+{% extends "registration/_registration_base.html" %}
+{% load url from future %}
+{% load i18n %}
+
+{% block content %}
+
+  {% if form.errors %}
+    <p>Your username and password didn't match. Please try again.</p>
+  {% endif %}
+
+  <form method="post" action="{% url 'django.contrib.auth.views.login' %}">
+    {% csrf_token %}
+    <table>
+      <tr>
+        <th>{{ form.username.label_tag }}</th>
+        <td>{{ form.username }}</td>
+      </tr>
+      <tr>
+        <th>{{ form.password.label_tag }}</th>
+        <td>{{ form.password }}</td>
+      </tr>
+      <tr>
+        <td></td>
+        <td>
+          <input class="btn pull-right" type="submit" value="Login" />
+        </td>
+      </tr>
+      <tr>
+        <td>&nbsp;</td>
+      </tr>
+      <tr>
+        <td>
+        </td>
+        <td>
+          <a class="pull-right" href="{% url "django.contrib.auth.views.password_reset" %}">Forgot password</a>
+        </td>
+      </tr>
+    </table>
+    <input type="hidden" name="next" value="/" />
+  </form>
+{% endblock %}

euscanwww/djeuscan/templates/registration/logout.html

@@ -0,0 +1,13 @@
+{% extends "registration/_registration_base.html" %}
+{% load i18n %}
+
+{% block content %}
+
+  <p>You're logged out."</p>
+  <p>Redirect...</p>
+
+  <script type="text/javascript">
+    window.location.href = "/";
+  </script>
+
+{% endblock %}

euscanwww/djeuscan/templates/registration/password_change_done.html

@@ -0,0 +1,6 @@
+{% extends "registration/_registration_base.html" %}
+{% block title %}Password changed{% endblock %}
+
+{% block content %}
+  Password successfully changed!
+{% endblock %}

euscanwww/djeuscan/templates/registration/password_change_form.html

@@ -0,0 +1,16 @@
+{% extends "registration/_registration_base.html" %}
+{% block title %}Change password{% endblock %}
+
+{% block content %}
+  <form method='post' action=''>{% csrf_token %}
+    <table>
+      {{ form }}
+      <tr>
+        <td></td>
+        <td>
+          <input type='submit' value="Change password" />
+        </td>
+      </tr>
+    </table>
+  </form>
+{% endblock %}

euscanwww/djeuscan/templates/registration/password_reset_complete.html

@@ -0,0 +1,7 @@
+{% extends "registration/_registration_base.html" %}
+
+{% block title %}Password reset complete{% endblock %}
+
+{% block content %}
+  Your password has been reset! You may now <a href="{{ login_url }}">log in</a>.
+{% endblock %}

euscanwww/djeuscan/templates/registration/password_reset_confirm.html

@@ -0,0 +1,15 @@
+{% extends "registration/_registration_base.html" %}
+{% block title %}Confirm password reset{% endblock %}
+
+{% block content %}
+  Enter your new password below to reset your password:
+  <form method="post" action="">{% csrf_token %}
+    <table>
+      {{ form.as_table }}
+      <tr>
+        <td></td>
+        <td><input type="submit" value="Set password" /></td>
+      </tr>
+    </table>
+  </form>
+{% endblock %}

euscanwww/djeuscan/templates/registration/password_reset_done.html

@@ -0,0 +1,10 @@
+{% extends "registration/_registration_base.html" %}
+
+{% block title %}Password reset{% endblock %}
+
+{% block content %}
+  <p>
+    We have sent you an email with a link to reset your password. Please check
+    your email and click the link to continue.
+  </p>
+{% endblock %}

euscanwww/djeuscan/templates/registration/password_reset_email.html

@@ -0,0 +1,16 @@
+Greetings {% if user.get_full_name %}{{ user.get_full_name }}{% else %}{{ user }}{% endif %},
+
+You are receiving this email because you (or someone pretending to be you)
+requested that your password be reset on the {{ domain }} site. If you do not
+wish to reset your password, please ignore this message.
+
+To reset your password, please click the following link, or copy and paste it
+into your web browser:
+
+{{ protocol }}://{{ domain }}{% url auth_password_reset_confirm uid token %}
+
+Your username, in case you've forgotten: {{ user.username }}
+
+
+Best regards,
+{{ site_name }} Management

euscanwww/djeuscan/templates/registration/password_reset_form.html

@@ -0,0 +1,18 @@
+{% extends "registration/_registration_base.html" %}
+
+{% block title %}Reset password{% endblock %}
+
+{% block content %}
+  Forgot your password? Enter your email in the form below and we'll send you
+  instructions for creating a new one.
+
+  <form method='post' action=''>{% csrf_token %}
+    <table>
+      {{ form }}
+      <tr>
+        <td></td>
+        <td><input type='submit' value="Reset password" /></td>
+      </tr>
+    </table>
+  </form>
+{% endblock %}

euscanwww/djeuscan/templates/registration/registration_complete.html

@@ -0,0 +1,6 @@
+{% extends "registration/_registration_base.html" %}
+{% block title %}Activation email sent{% endblock %}
+
+{% block content %}
+  An activation email has been sent. Please check your email and click on the link to activate your account.
+{% endblock %}

euscanwww/djeuscan/templates/registration/registration_form.html

@@ -0,0 +1,19 @@
+{% extends "registration/_registration_base.html" %}
+
+{% block title %}
+  Register for an account
+{% endblock %}
+
+{% block content %}
+  <table>
+    <form method='post' action=''>{% csrf_token %}
+      {{ form }}
+      <tr>
+        <td></td>
+        <td>
+          <input class="btn pull-right" type="submit" value="Send activation email" />
+        </td>
+      </tr>
+    </form>
+  </table>
+{% endblock %}

euscanwww/djeuscan/templatetags/euscan.py

@@ -0,0 +1,49 @@
+from django import template
+from django.conf import settings
+
+register = template.Library()
+
+
+@register.inclusion_tag('euscan/_packages.html', takes_context=True)
+def packages(context, pkgs):
+    context['packages'] = pkgs
+    return context
+
+
+@register.inclusion_tag('euscan/_package_cols.html', takes_context=True)
+def package_cols(context, infos):
+    context['infos'] = infos
+    return context
+
+
+@register.inclusion_tag('euscan/_package_bar.html', takes_context=True)
+def package_bar(context, infos):
+    context['infos'] = infos
+    return context
+
+
+@register.inclusion_tag('euscan/_categories_table.html')
+def categories_table(categories, extras=False):
+    return {
+        "categories": categories,
+        "extras": extras,
+        "STATIC_URL": settings.STATIC_URL,
+    }
+
+
+@register.inclusion_tag('euscan/_herds_table.html')
+def herds_table(herds, extras=False):
+    return {
+        "herds": herds,
+        "extras": extras,
+        "STATIC_URL": settings.STATIC_URL,
+    }
+
+
+@register.inclusion_tag('euscan/_maintainers_table.html')
+def maintainers_table(maintainers, extras=False):
+    return {
+        "maintainers": maintainers,
+        "extras": extras,
+        "STATIC_URL": settings.STATIC_URL,
+    }

euscanwww/djeuscan/templatetags/packages.py

@@ -1,21 +0,0 @@
-from django import template
-
-register = template.Library()
-
-
-@register.inclusion_tag('euscan/_packages.html', takes_context=True)
-def packages(context, pkgs):
-    context['packages'] = pkgs
-    return context
-
-
-@register.inclusion_tag('euscan/_package_cols.html', takes_context=True)
-def package_cols(context, infos):
-    context['infos'] = infos
-    return context
-
-
-@register.inclusion_tag('euscan/_package_bar.html', takes_context=True)
-def package_bar(context, infos):
-    context['infos'] = infos
-    return context

euscanwww/djeuscan/urls.py

@@ -1,12 +1,26 @@
 from django.conf.urls.defaults import url, patterns, include
+from django.contrib.auth.decorators import user_passes_test
+
+from djcelery.views import apply as apply_task
+from djeuscan.views import registered_tasks
+
 from djeuscan.feeds import PackageFeed, CategoryFeed, HerdFeed, \
     MaintainerFeed, GlobalFeed
 
+
+admin_required = user_passes_test(lambda u: u.is_superuser)
+
+
 package_patterns = patterns('djeuscan.views',
     url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/feed/$',
         PackageFeed(), name='package_feed'),
     url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/$',
         'package', name="package"),
+    url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/favourite$',
+        'favourite_package', name="favourite_package"),
+    url((r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/'
+         'unfavourite$'),
+        'unfavourite_package', name="unfavourite_package"),
 )
 
 categories_patterns = patterns('djeuscan.views',
@@ -16,6 +30,10 @@ categories_patterns = patterns('djeuscan.views',
         name='category_feed'),
     url(r'^(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$',
         'chart_category', name="chart_category"),
+    url(r'^(?P<category>[\w+][\w+.-]*)/favourite$',
+        'favourite_category', name="favourite_category"),
+    url(r'^(?P<category>[\w+][\w+.-]*)/unfavourite$',
+        'unfavourite_category', name="unfavourite_category"),
     url(r'^$', 'categories', name="categories"),
 )
 
@@ -24,6 +42,10 @@ herds_patterns = patterns('djeuscan.views',
     url(r'^(?P<herd>[\@\{\}\w+.-]*)/feed/$', HerdFeed(), name='herd_feed'),
     url(r'^(?P<herd>[\@\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$',
         'chart_herd', name="chart_herd"),
+    url(r'^(?P<herd>[\@\{\}\w+.-]*)/favourite$', 'favourite_herd',
+        name="favourite_herd"),
+    url(r'^(?P<herd>[\@\{\}\w+.-]*)/unfavourite$', 'unfavourite_herd',
+        name="unfavourite_herd"),
     url(r'^$', 'herds', name="herds"),
 )
 
@@ -33,6 +55,10 @@ maintainers_patterns = patterns('djeuscan.views',
         name='maintainer_feed'),
     url(r'^(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$',
         'chart_maintainer', name="chart_maintainer"),
+    url(r'^(?P<maintainer_id>\d+)/favourite$',
+        'favourite_maintainer', name="favourite_maintainer"),
+    url(r'^(?P<maintainer_id>\d+)/unfavourite$',
+        'unfavourite_maintainer', name="unfavourite_maintainer"),
     url(r'^$', 'maintainers', name="maintainers"),
 )
 
@@ -41,6 +67,26 @@ overlays_patterns = patterns('djeuscan.views',
     url(r'^$', 'overlays', name="overlays"),
 )
 
+tasks_patterns = patterns('djeuscan.views',
+    url(r'^refresh_package/(?P<query>(?:[\w+][\w+.-]*/[\w+][\w+.-]*))/$',
+        "refresh_package",
+        name="refresh_package"),
+    url(r'^registered_tasks/$', admin_required(registered_tasks),
+        name="registered_tasks"),
+    url(r'^apply/(?P<task_name>.*)/$', admin_required(apply_task),
+        name="apply_task"),
+)
+
+accounts_patterns = patterns('djeuscan.views',
+    url(r'^profile/$', 'accounts_index', name="accounts_index"),
+    url(r'^categories/$', 'accounts_categories', name="accounts_categories"),
+    url(r'^herds/$', 'accounts_herds', name="accounts_herds"),
+    url(r'^maintainers/$', 'accounts_maintainers',
+        name="accounts_maintainers"),
+    url(r'^packages/$', 'accounts_packages', name="accounts_packages"),
+)
+
+
 urlpatterns = patterns('djeuscan.views',
     # Global stuff
     url(r'^api/', include('djeuscan.api.urls')),
@@ -60,4 +106,7 @@ urlpatterns = patterns('djeuscan.views',
     url(r'^maintainers/', include(maintainers_patterns)),
     url(r'^overlays/', include(overlays_patterns)),
     url(r'^package/', include(package_patterns)),
+
+    url(r'^tasks/', include(tasks_patterns)),
+    url(r'^accounts/', include(accounts_patterns)),
 )

euscanwww/djeuscan/views.py

@@ -1,13 +1,19 @@
 """ Views """
 
-from annoying.decorators import render_to
+import inspect
+from annoying.decorators import render_to, ajax_request
+
 from django.http import Http404
 from django.shortcuts import get_object_or_404
+from django.contrib.auth.decorators import login_required
+from django.views.decorators.http import require_POST
 
 from djeuscan.helpers import version_key, packages_from_names
 from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
-    VersionLog
+    VersionLog, RefreshPackageQuery, HerdAssociation, MaintainerAssociation, \
+    CategoryAssociation, PackageAssociation
 from djeuscan.forms import WorldForm, PackagesForm
+from djeuscan.tasks import admin_tasks
 from djeuscan import charts
 
 
@@ -161,13 +167,23 @@ def package(request, category, package):
     except EuscanResult.DoesNotExist:
         last_scan = None
 
+    favourited = False
+    if request.user.is_authenticated():
+        try:
+            PackageAssociation.objects.get(user=request.user, package=package)
+        except PackageAssociation.DoesNotExist:
+            pass
+        else:
+            favourited = True
+
     return {
         'package': package,
         'packaged': packaged,
         'upstream': upstream,
         'log': log,
         'vlog': vlog,
-        'last_scan': last_scan
+        'last_scan': last_scan,
+        'favourited': favourited,
     }
 
 
@@ -254,3 +270,164 @@ def chart_herd(request, **kwargs):
 
 def chart_category(request, **kwargs):
     return chart(request, **kwargs)
+
+
+@ajax_request
+def registered_tasks(request):
+    data = {}
+    for task in admin_tasks:
+        argspec = inspect.getargspec(task.run)
+        data[task.name] = {
+            "args": argspec.args,
+            "defaults": argspec.defaults,
+            "default_types": None
+        }
+        if argspec.defaults is not None:
+            data[task.name].update({
+                "defaults_types": [type(x).__name__ for x in argspec.defaults]
+            })
+    return {"tasks": data}
+
+
+@login_required
+@require_POST
+@ajax_request
+def refresh_package(request, query):
+    obj, created = RefreshPackageQuery.objects.get_or_create(query=query)
+    if not created:
+        obj.priority += 1
+        obj.save()
+    return {"result": "success"}
+
+
+@login_required
+@render_to('euscan/accounts/index.html')
+def accounts_index(request):
+    return {}
+
+
+@login_required
+@render_to('euscan/accounts/categories.html')
+def accounts_categories(request):
+    categories = [obj.category for obj in
+                  CategoryAssociation.objects.filter(user=request.user)]
+    return {"categories": categories}
+
+
+@login_required
+@render_to('euscan/accounts/herds.html')
+def accounts_herds(request):
+    herds = [obj.herd for obj in
+             HerdAssociation.objects.filter(user=request.user)]
+    return {"herds": herds}
+
+
+@login_required
+@render_to('euscan/accounts/maintainers.html')
+def accounts_maintainers(request):
+    maintainers = [obj.maintainer for obj in
+                   MaintainerAssociation.objects.filter(user=request.user)]
+    return {"maintainers": maintainers}
+
+
+@login_required
+@render_to('euscan/accounts/packages.html')
+def accounts_packages(request):
+    packages = [obj.package for obj in
+                PackageAssociation.objects.filter(user=request.user)]
+    return {"packages": packages}
+
+
+@login_required
+@require_POST
+@ajax_request
+def favourite_package(request, category, package):
+    obj = get_object_or_404(Package, category=category, name=package)
+    _, created = PackageAssociation.objects.get_or_create(
+        user=request.user, package=obj
+    )
+    return {"success": created}
+
+
+@login_required
+@require_POST
+@ajax_request
+def unfavourite_package(request, category, package):
+    package = get_object_or_404(Package, category=category, name=package)
+    obj = get_object_or_404(
+        PackageAssociation, package=package, user=request.user
+    )
+    obj.delete()
+    return {"success": True}
+
+
+@login_required
+@require_POST
+@ajax_request
+def favourite_herd(request, herd):
+    obj = get_object_or_404(Herd, herd=herd)
+    _, created = HerdAssociation.objects.get_or_create(
+        user=request.user, herd=obj
+    )
+    return {"success": created}
+
+
+@login_required
+@require_POST
+@ajax_request
+def unfavourite_herd(request, herd):
+    herd = get_object_or_404(Herd, herd=herd)
+    obj = get_object_or_404(
+        HerdAssociation, herd=herd, user=request.user
+    )
+    obj.delete()
+    return {"success": True}
+
+
+@login_required
+@require_POST
+@ajax_request
+def favourite_maintainer(request, maintainer_id):
+    obj = get_object_or_404(Maintainer, pk=maintainer_id)
+    _, created = MaintainerAssociation.objects.get_or_create(
+        user=request.user, maintainer=obj
+    )
+    return {"success": created}
+
+
+@login_required
+@require_POST
+@ajax_request
+def unfavourite_maintainer(request, maintainer_id):
+    maintainer = get_object_or_404(Maintainer, pk=maintainer_id)
+    obj = get_object_or_404(
+        MaintainerAssociation, maintainer=maintainer, user=request.user
+    )
+    obj.delete()
+    return {"success": True}
+
+
+@login_required
+@require_POST
+@ajax_request
+def favourite_category(request, category):
+    packages = Package.objects.for_category(category, last_versions=True)
+
+    if not packages:
+        raise Http404
+
+    _, created = CategoryAssociation.objects.get_or_create(
+        user=request.user, category=category
+    )
+    return {"success": created}
+
+
+@login_required
+@require_POST
+@ajax_request
+def unfavourite_category(request, category):
+    obj = get_object_or_404(
+        CategoryAssociation, user=request.user, category=category
+    )
+    obj.delete()
+    return {"success": True}

euscanwww/euscanwww/settings.py

@@ -157,6 +157,9 @@ TEMPLATE_CONTEXT_PROCESSORS = (
 )
 
 INSTALLED_APPS = (
+    'euscanwww',
+    'djeuscan',
+
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
@@ -166,8 +169,8 @@ INSTALLED_APPS = (
     # Uncomment the next line to enable admin documentation:
     # 'django.contrib.admindocs',
     'south',
-    'euscanwww',
+    'djcelery',
-    'djeuscan',
+    'registration',
 )
 
 # A sample logging configuration. The only tangible logging
@@ -178,12 +181,22 @@ INSTALLED_APPS = (
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
+    'formatters': {
+        'simple': {
+            'format': '%(levelname)s %(asctime)s %(message)s'
+        },
+    },
     'filters': {
         'require_debug_false': {
             '()': 'django.utils.log.RequireDebugFalse'
         }
     },
     'handlers': {
+        'console': {
+            'level': 'INFO',
+            'class': 'logging.StreamHandler',
+            'formatter': 'simple'
+        },
         'mail_admins': {
             'level': 'ERROR',
             'filters': ['require_debug_false'],
@@ -196,14 +209,49 @@ LOGGING = {
             'level': 'ERROR',
             'propagate': True,
         },
+        'djeuscan': {
+            'handlers': ['console'],
+            'level': 'INFO',
+            'propagate': True
+        }
     }
 }
 
+# django-registration
+ACCOUNT_ACTIVATION_DAYS = 7
+EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
+
+# djeuscan tasks
+PORTAGE_ROOT = "/usr/portage/"
+PORTAGE_CONFIGROOT = PORTAGE_ROOT
+LAYMAN_CONFIG = "/etc/layman/layman.cfg"
+EMERGE_REGEN_JOBS = 4
+
+# Celery config
+import djcelery
+djcelery.setup_loader()
+BROKER_URL = "amqp://guest:guest@localhost:5672//"
+CELERY_RESULT_BACKEND = "amqp"
+BROKER_CONNECTION_TIMEOUT = 3600
+CELERYD_CONCURRENCY = 4
+
+TASKS_CONCURRENTLY = 4
+TASKS_SUBTASK_PACKAGES = 32
+
+# LDAP authentication
+# TODO: Test data - change me!
+AUTH_LDAP_SERVER_URI = "ldap://localhost"
+AUTH_LDAP_USER_DN_TEMPLATE = "uid=%(user)s,ou=users,dc=my-domain,dc=com"
+AUTHENTICATION_BACKENDS = (
+    'django_auth_ldap.backend.LDAPBackend',
+    'django.contrib.auth.backends.ModelBackend',
+)
+
 try:
     from local_settings import *
 except ImportError, ex:
     import sys
-    sys.stderr.write(\
+    sys.stderr.write(
-            ("settings.py: error importing local settings file:\n" + \
+        "settings.py: error importing local settings file:\n"
-            "\t%s\n" + \
+        "\t%s\nDo you have a local_settings.py module?\n" % str(ex)
-            "Do you have a local_settings.py module?\n") % str(ex))
+    )

euscanwww/euscanwww/urls.py

@@ -10,6 +10,7 @@ urlpatterns = patterns('',
     # (r'^admin/doc/', include('django.contrib.admindocs.urls')),
 
     url(r'^admin/', include(admin.site.urls)),
+    url(r'^accounts/', include('registration.backends.default.urls')),
     url(r'^', include('djeuscan.urls')),
 )
 

euscanwww/htdocs/css/style.css

@@ -1,96 +1,80 @@
-body
+body {
-{
-  margin: 0;
-  padding: 0;
     font-size: 0.8em;
     font-family: Dejavu, Verdana, "Bitstream Vera Sans", "Lucida Grande", "Trebuchet MS", sans-serif;
     color: #535353;
     background: #D2D0D4;
-  }
-
-img
-{
-  border: none;
-  }
-
-h1
-{
-  margin-top: 0;
-  color: #369;
-  font-size: 1.6em;
 }
 
-a:link, a:visited
+img {
-{
+    border: none;
+}
+
+h1 {
+    margin-top: 0;
+    color: #333;
+    font-size: 1.6em;
+    width: 20em;
+    margin-left: auto;
+    margin-right: auto;
+}
+
+a:link, a:visited {
     text-decoration: none;
     font-weight: bold;
     color: #ff8c00;
-  }
+}
 
-a:hover, a:active
+a:hover, a:active {
-{
     font-weight: bold;
     color: #ff4500;
     text-decoration: underline;
-  }
+}
 
 #header {
-    /* background: url(http://packages.gentoo.org/media/packages_gentoo_logo.jpg) no-repeat;
-    background-color: #46347C; */
     width: 100%;
     height: 30px;
     margin: 0;
     padding: 0;
     background: url(../img/gentoo-header-bar-bg.png) repeat-x;
+    margin-bottom: 20px;
 }
 
-#header #logo
+#header #logo {
-{
     float: left;
 }
 
 #content {
-    background: #F0F0F0;
-    padding: 10px;
-    margin-top: 10px;
-/*    margin-right: auto;
-    margin-left: auto;
-*/
-    margin-left: 20px;
-    /* margin-left: 20%; */
-    margin-right: 250px;
-    /* max-width: 60em; */
-
     border: 1px solid #67539B;
     background: #FEFEFE;
 
-    -webkit-border-radius: 10px;
+    padding: 20px;
-    -moz-border-radius: 10px;
+
-    border-radius: 10px;
+    min-height: 350px;
 }
 
 .menu {
-    -webkit-border-radius: 10px;
-    -moz-border-radius: 10px;
-    border-radius: 10px;
     border: 1px solid #67539B;
-}
-
-.menu {
-    width:100%;
-    margin-top: 30px;
     background: #8076A1;
     color: #FFF;
+    padding-top: 5px;
+    padding-bottom: 5px;
+}
+
+.notfirst-menu {
+    margin-top: 10px;
 }
 
 .menu a {
     color: #fff;
 }
 
-.menu li
+.menu li {
-{
     list-style-type: none;
- }
+}
+
+.submenu {
+    padding-left: 15px;
+}
 
 .menu dl {
     margin-left: 15px;
@@ -100,29 +84,7 @@ a:hover, a:active
     border: none;
 }
 
-#menus {
+a {
-    width: 200px;
-    position:absolute;
-    top:30px;
-    right:20px;
-
-/*
-    position: fixed;
-    right: 10px;
-    top: 2em;
-    width: 20%;
-*/
-}
-
-/*
-code, pre{
-	background-color:transparent;
-	font-family:"Courier New",Courier,monospace;
-	font-size:small;
-}
-*/
-
-a{
     color: #3F4C66;
 }
 
@@ -139,7 +101,7 @@ abbr:hover {
     text-decoration: underline;
 }
 
-pre{
+pre {
     border-left:5px solid;
     padding:0.5em 1em;
     margin-left:2em;
@@ -155,29 +117,15 @@ dd {
     padding: 5px;
 }
 
-
+hr {
-h1 {
-        color: #000;
-	width: 20em;
-	margin-left: auto;
-	margin-right: auto;
-}
-
-.ok {
-	color:#15B100;
-}
-
-hr
-{
     margin: 0.3em 1em 0.3em 1em;
     height: 1px;
     border: #bcbcbc dashed;
     border-width: 0 0 1px 0;
-  }
+}
 
 
-table
+table {
-{
     max-width: 60em;
     /* width: 50%; */
     border-collapse: collapse;
@@ -188,42 +136,38 @@ table
     text-align: left;
 }
 
-td
+td {
-{
     vertical-align: top;
-  }
+}
 
-th
+th {
-{
     font-size: 0.8em;
     text-align: center;
     border-bottom: 3px solid;
- }
+}
 
-#footer
+#footer {
-{
     background: #8076A1;
     font-size: 0.8em;
     width: 100%;
     margin: 0;
-    margin-top: 10px;
+    margin-top: 20px;
     color: #FFF;
- }
+}
 
 #footer p {
     text-align: right;
     margin: 5px;
 }
 
-.err,.ok,.inf
+.err, .ok, .inf {
-{
     margin: 5px;
     margin-left: auto;
     margin-right: auto;
     width: 70%;
     font-weight:bold;
     border: 1px dotted #5682AD;
-  }
+}
 
 .added {
     color: #262;
@@ -233,17 +177,15 @@ th
     color: #F00;
 }
 
-.err
+.err {
-{
     border-color: #F00;
     color: #F00;
- }
+}
 
-.ok
+.ok {
-{
     border-color: #262;
     color: #262;
-  }
+}
 
 .logo {
     float: right;
@@ -281,3 +223,13 @@ th
     max-height: 100pt;
     overflow: auto;
 }
+
+.hide {
+    display: none;
+}
+
+.rounded {
+    -webkit-border-radius: 10px;
+    -moz-border-radius: 10px;
+    border-radius: 10px;
+}

euscanwww/scripts/euscan.fcgi

@@ -1,6 +1,7 @@
 #!/usr/bin/python
 
-import sys, os
+import sys
+import os
 import os.path
 
 PROJECT = '/path/to/euscanwww'

pym/euscan/__init__.py

@@ -19,7 +19,8 @@ CONFIG = {
     'skip-robots-txt': False,
     'cache': False,
     'format': None,
-    'indent': 2
+    'indent': 2,
+    'progress': False
 }
 
 BLACKLIST_VERSIONS = [
@@ -67,4 +68,4 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [
 ]
 
 from out import EuscanOutput
-output = out.EuscanOutput(CONFIG)
+output = EuscanOutput(CONFIG)

pym/euscan/handlers/__init__.py

@@ -1,6 +1,17 @@
-from euscan.handlers import generic, php, pypi, rubygem, kde, cpan, github
+import pkgutil
 
-handlers = [kde, php, pypi, rubygem, cpan, github, generic]
+# autoimport all modules in this directory and append them to handlers list
+handlers = []
+for loader, module_name, is_pkg in  pkgutil.walk_packages(__path__):
+    module = loader.find_module(module_name).load_module(module_name)
+    handlers.append(module)
+
+# sort handlers by priority (e.g.: generic should be run lastly)
+handlers = sorted(
+    handlers,
+    key=lambda handler: handler.PRIORITY,
+    reverse=True
+)
 
 
 def find_best_handler(cpv, url):

pym/euscan/handlers/cpan.py

@@ -7,6 +7,7 @@ from euscan import helpers, output
 
 HANDLER_NAME = "cpan"
 CONFIDENCE = 100.0
+PRIORITY = 100
 
 _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*")
 

pym/euscan/handlers/generic.py

@@ -14,6 +14,7 @@ from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \
 
 HANDLER_NAME = "generic"
 CONFIDENCE = 50.0
+PRIORITY = 0
 
 BRUTEFORCE_HANDLER_NAME = "brute_force"
 BRUTEFORCE_CONFIDENCE = 30.0

pym/euscan/handlers/github.py

@@ -1,4 +1,5 @@
-import json, urllib2
+import json
+import urllib2
 import re
 
 import portage
@@ -7,16 +8,20 @@ from euscan import helpers, output
 
 HANDLER_NAME = "github"
 CONFIDENCE = 100.0
+PRIORITY = 100
+
 
 def can_handle(cpv, url):
     return url.startswith('mirror://github/')
 
+
 def guess_package(cp, url):
     match = re.search('^mirror://github/(.*?)/(.*?)/(.*)$', url)
 
     assert(match)
     return (match.group(1), match.group(2), match.group(3))
 
+
 def scan(cpv, url):
     'http://developer.github.com/v3/repos/downloads/'
 
@@ -30,11 +35,13 @@ def scan(cpv, url):
     # now create a filename-matching regexp
     # XXX: supposedly replace first with (?P<foo>...)
     # and remaining ones with (?P=foo)
-    fnre = re.compile('^%s$' % re.escape(filename).replace(re.escape(ver), '(.*?)'))
+    fnre = re.compile('^%s$' % \
+                      re.escape(filename).replace(re.escape(ver), '(.*?)'))
 
-    output.einfo("Using github API for: " + '/'.join(package))
+    output.einfo("Using github API for: " + '/'.join(filename))
 
-    dlreq = urllib2.urlopen('https://api.github.com/repos/%s/%s/downloads' % (user, project))
+    dlreq = urllib2.urlopen('https://api.github.com/repos/%s/%s/downloads' % \
+                            (user, project))
     dls = json.load(dlreq)
 
     for dl in dls:
@@ -46,5 +53,6 @@ def scan(cpv, url):
                 continue
             yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE)
 
+
 def brute_force(cpv, url):
     return []

pym/euscan/handlers/kde.py

@@ -1,5 +1,7 @@
 from euscan.handlers import generic
 
+PRIORITY = 100
+
 HANDLER_NAME = "kde"
 
 

pym/euscan/handlers/php.py

@@ -7,6 +7,7 @@ from euscan import helpers, output
 
 HANDLER_NAME = "php"
 CONFIDENCE = 100.0
+PRIORITY = 100
 
 
 def can_handle(cpv, url):

pym/euscan/handlers/pypi.py

@@ -7,6 +7,7 @@ from euscan import helpers, output
 
 HANDLER_NAME = "pypi"
 CONFIDENCE = 100.0
+PRIORITY = 100
 
 
 def can_handle(cpv, url):

pym/euscan/handlers/rubygem.py

@@ -7,6 +7,7 @@ from euscan import helpers, output
 
 HANDLER_NAME = "rubygem"
 CONFIDENCE = 100.0
+PRIORITY = 100
 
 
 def can_handle(cpv, url):

pym/euscan/helpers.py

@@ -44,7 +44,7 @@ def get_version_type(version):
         if token in gentoo_types:
             types.append(token)
     if types:
-        return types[0]
+        return types[0]  # TODO: consider returning all types
     return "release"
 
 

pym/euscan/out.py

@@ -3,6 +3,7 @@ from collections import defaultdict
 import json
 import signal
 import time
+import re
 
 from gentoolkit import pprinter as pp
 import portage
@@ -10,15 +11,20 @@ from portage.output import EOutput, TermProgressBar
 
 
 class ProgressHandler(object):
-    def __init__(self):
+    def __init__(self, progress_bar):
         self.curval = 0
         self.maxval = 0
         self.last_update = 0
         self.min_display_latency = 0.2
+        self.progress_bar = progress_bar
+
+    def on_progress(self, maxval=None, increment=1, label=None):
+        self.maxval = maxval or self.maxval
+        self.curval += increment
+
+        if label:
+            self.progress_bar.label(label)
 
-    def on_progress(self, maxval, curval):
-        self.maxval = maxval
-        self.curval = curval
         cur_time = time.time()
         if cur_time - self.last_update >= self.min_display_latency:
             self.last_update = cur_time
@@ -32,7 +38,7 @@ def progress_bar():
     on_progress = None
     progress_bar = TermProgressBar()
 
-    progress_handler = ProgressHandler()
+    progress_handler = ProgressHandler(progress_bar)
     on_progress = progress_handler.on_progress
 
     def display():
@@ -61,12 +67,20 @@ class EOutputMem(EOutput):
         self.out = StringIO()
 
     def getvalue(self):
-        return self.out.getvalue()
+        return clean_colors(self.out.getvalue())
 
     def _write(self, f, msg):
         super(EOutputMem, self)._write(self.out, msg)
 
 
+def clean_colors(string):
+    if type(string) is str:
+        string = re.sub("\033\[[0-9;]+m", "", string)
+        string = re.sub(r"\\u001b\[[0-9;]+m", "", string)
+        string = re.sub(r"\x1b\[[0-9;]+m", "", string)
+    return string
+
+
 class EuscanOutput(object):
     """
     Class that handles output for euscan
@@ -76,6 +90,10 @@ class EuscanOutput(object):
         self.queries = defaultdict(dict)
         self.current_query = None
 
+    def clean(self):
+        self.queries = defaultdict(dict)
+        self.current_query = None
+
     def set_query(self, query):
         self.current_query = query
         if query is None:
@@ -95,7 +113,7 @@ class EuscanOutput(object):
             "metadata": {},
         }
 
-    def get_formatted_output(self):
+    def get_formatted_output(self, format_=None):
         data = {}
 
         for query in self.queries:
@@ -105,8 +123,11 @@ class EuscanOutput(object):
                 "messages": self.queries[query]["output"].getvalue(),
             }
 
-        if self.config["format"].lower() == "json":
+        format_ = format_ or self.config["format"]
+        if format_.lower() == "json":
             return json.dumps(data, indent=self.config["indent"])
+        elif format_.lower() == "dict":
+            return data
         else:
             raise TypeError("Invalid output format")
 
@@ -128,12 +149,13 @@ class EuscanOutput(object):
 
     def metadata(self, key, value, show=True):
         if self.config["format"]:
-            self.queries[self.current_query]["metadata"][key] = value
+            self.queries[self.current_query]["metadata"][key] = \
+                clean_colors(value)
         elif show:
             print "%s: %s" % (key.capitalize(), value)
 
     def __getattr__(self, key):
-        if not self.config["quiet"]:
+        if not self.config["quiet"] and self.current_query is not None:
             output = self.queries[self.current_query]["output"]
             return getattr(output, key)
         else:

pym/euscan/scan.py

@@ -1,4 +1,7 @@
+from __future__ import print_function
+
 import os
+import sys
 from datetime import datetime
 
 import portage
@@ -42,15 +45,21 @@ def filter_versions(cp, versions):
 def scan_upstream_urls(cpv, urls, on_progress):
     versions = []
 
-    maxval = len(urls) + 5
+    if on_progress:
-    curval = 1
+        progress_available = 70
+        num_urls = sum([len(urls[fn]) for fn in urls])
+        if num_urls > 0:
+            progress_increment = progress_available / num_urls
+        else:
+            progress_increment = 0
 
     for filename in urls:
-        curval += 1
-        if on_progress:
-            on_progress(maxval, curval)
-
         for url in urls[filename]:
+
+            if on_progress and progress_available > 0:
+                on_progress(increment=progress_increment)
+                progress_available -= progress_increment
+
             if not CONFIG['quiet'] and not CONFIG['format']:
                 pp.uprint()
             output.einfo("SRC_URI is '%s'" % url)
@@ -79,15 +88,10 @@ def scan_upstream_urls(cpv, urls, on_progress):
 
     cp, ver, rev = portage.pkgsplit(cpv)
 
-    curval += 1
-    if on_progress:
-        on_progress(maxval, curval)
-
     result = filter_versions(cp, versions)
 
-    curval += 1
+    if on_progress and progress_available > 0:
-    if on_progress:
+        on_progress(increment=progress_available)
-        on_progress(maxval, curval)
 
     return result
 
@@ -115,9 +119,6 @@ def scan_upstream(query, on_progress=None):
     Scans the upstream searching new versions for the given query
     """
 
-    maxval = 3
-    curval = 0
-
     matches = []
 
     if query.endswith(".ebuild"):
@@ -156,9 +157,8 @@ def scan_upstream(query, on_progress=None):
     output.metadata("cp", pkg.cp, show=False)
     output.metadata("cpv", pkg.cpv, show=False)
 
-    curval += 1
     if on_progress:
-        on_progress(maxval, curval)
+        on_progress(increment=10)
 
     if pkg.cp in BLACKLIST_PACKAGES:
         output.ewarn(
@@ -210,14 +210,16 @@ def scan_upstream(query, on_progress=None):
     scan_time = (datetime.now() - start_time).total_seconds()
     output.metadata("scan_time", scan_time, show=False)
 
-    curval += 1
-    if on_progress:
-        on_progress(maxval, curval)
-
     result = scan_upstream_urls(pkg.cpv, urls, on_progress)
 
-    curval += 1
     if on_progress:
-        on_progress(maxval, curval)
+        on_progress(increment=10)
+
+    if len(result) > 0:
+        if not (CONFIG['format'] or CONFIG['quiet']):
+            print("\n", file=sys.stderr)
+
+        for cp, url, version, handler, confidence in result:
+            output.result(cp, version, url, handler, confidence)
 
     return result

setup.py

@@ -87,9 +87,12 @@ setup(
         'https://github.com/iksaif/euscan/tarball/' +
         ('master' if __version__ == '9999' else ('euscan-%s' % __version__))
     ),
-    install_requires=['Django==1.4', 'django-annoying==0..7.6', 'South==0.7.4',
+    install_requires=[
-                      'django-piston==0.2.3', 'BeautifulSoup==3.2.1',
+        'Django==1.4', 'django-annoying==0.7.6', 'South==0.7.4',
-                      'matplotlib==1.1.0', 'django-celery==2.5.5'],
+        'django-piston==0.2.3', 'BeautifulSoup==3.2.1', 'matplotlib==1.1.0',
+        'django-celery==2.5.5', 'django-registration==0.8',
+        'python-ldap==2.4.10', 'django-auth-ldap==1.1',
+    ],
     package_dir={'': 'pym'},
     packages=packages,
     package_data={},