From 56c4d79fb2c30a3481c80693875f24c9a871bd51 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Wed, 30 May 2012 22:54:55 +0200
Subject: [PATCH 01/37] euscanwww: starting implementing Celery tasks

Basic tasks.py module, some fixes in the management commands

Signed-off-by: volpino <fox91@anche.no>
---
 .../management/commands/scan_metadata.py      |  8 +-
 .../management/commands/scan_portage.py       | 21 ++---
 .../management/commands/scan_upstream.py      | 49 +++++------
 euscanwww/djeuscan/tasks.py                   | 83 +++++++++++++++++++
 euscanwww/euscanwww/settings.py               |  9 ++
 5 files changed, 126 insertions(+), 44 deletions(-)
 create mode 100644 euscanwww/djeuscan/tasks.py

diff --git a/euscanwww/djeuscan/management/commands/scan_metadata.py b/euscanwww/djeuscan/management/commands/scan_metadata.py
index 8590412..d9c7e49 100644
--- a/euscanwww/djeuscan/management/commands/scan_metadata.py
+++ b/euscanwww/djeuscan/management/commands/scan_metadata.py
@@ -18,7 +18,7 @@ class ScanMetadata(object):
         self.style = color_style()
 
     @commit_on_success
-    def run(self, query=None, obj=None):
+    def scan(self, query=None, obj=None):
         matches = Query(query).find(
             include_masked=True,
             in_installed=False,
@@ -158,10 +158,10 @@ class Command(BaseCommand):
 
         if options['all']:
             for pkg in Package.objects.all():
-                scan_metadata.run('%s/%s' % (pkg.category, pkg.name), pkg)
+                scan_metadata.scan('%s/%s' % (pkg.category, pkg.name), pkg)
         elif len(args) > 0:
             for package in args:
-                scan_metadata.run(package)
+                scan_metadata.scan(package)
         else:
             for package in sys.stdin.readlines():
-                scan_metadata.run(package[:-1])
+                scan_metadata.scan(package[:-1])
diff --git a/euscanwww/djeuscan/management/commands/scan_portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py
index 4227501..9533730 100644
--- a/euscanwww/djeuscan/management/commands/scan_portage.py
+++ b/euscanwww/djeuscan/management/commands/scan_portage.py
@@ -3,8 +3,8 @@ import portage
 import sys
 import os
 import re
-
 from optparse import make_option
+from collections import defaultdict
 
 from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand
@@ -14,13 +14,10 @@ from djeuscan.models import Package, Version, VersionLog
 
 
 class ScanPortage(object):
-    def __init__(self, stdout=None, **options):
-        if stdout is None:
-            self.stdout = sys.stdout
-        else:
-            self.stdout = stdout
+    def __init__(self, stdout=None, options=None):
+        self.stdout = sys.stdout if stdout is None else stdout
+        self.options = defaultdict(None) if options is None else options
 
-        self.options = options
         self.style = color_style()
         self._cache = {'packages': {}, 'versions': {}}
         self._overlays = None
@@ -84,7 +81,7 @@ class ScanPortage(object):
         return self._overlays
 
     @commit_on_success
-    def run(self, query=None):
+    def scan(self, query=None):
         env = os.environ
         env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
 
@@ -312,7 +309,7 @@ class Command(BaseCommand):
     help = 'Scans portage tree and fills database'
 
     def handle(self, *args, **options):
-        scan_portage = ScanPortage(stdout=self.stdout, **options)
+        scan_portage = ScanPortage(stdout=self.stdout, options=options)
 
         if not options['quiet']:
             self.stdout.write('Scanning portage tree...\n')
@@ -329,13 +326,13 @@ class Command(BaseCommand):
                 self.stdout.write('done\n')
 
         if options['all']:
-            scan_portage.run()
+            scan_portage.scan()
         elif len(args):
             for package in args:
-                scan_portage.run(package)
+                scan_portage.scan(package)
         else:
             for package in sys.stdin.readlines():
-                scan_portage.run(package[:-1])
+                scan_portage.scan(package[:-1])
 
         if options['purge-versions']:
             purge_versions(options)
diff --git a/euscanwww/djeuscan/management/commands/scan_upstream.py b/euscanwww/djeuscan/management/commands/scan_upstream.py
index f6e78c3..2e2d9c2 100644
--- a/euscanwww/djeuscan/management/commands/scan_upstream.py
+++ b/euscanwww/djeuscan/management/commands/scan_upstream.py
@@ -4,7 +4,6 @@ import sys
 import re
 from StringIO import StringIO
 from optparse import make_option
-from collections import defaultdict
 
 from django.utils import timezone
 from django.db.transaction import commit_on_success
@@ -14,21 +13,17 @@ from djeuscan.models import Package, Version, EuscanResult, VersionLog
 
 
 class ScanUpstream(object):
-    def __init__(self, options=None):
-        if options is None:
-            self.options = defaultdict(None)
-        else:
-            self.options = options
+    def __init__(self, quiet=False):
+        self.quiet = quiet
 
-    def run(self, packages=None):
-        for package in packages:
-            cmd = ['euscan', package]
+    def scan(self, package):
+        cmd = ['euscan', package]
 
-            fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                                  stderr=subprocess.PIPE)
-            output = StringIO(fp.communicate()[0])
+        fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+        output = StringIO(fp.communicate()[0])
 
-            self.parse_output(output)
+        self.parse_output(output)
 
     def store_result(self, package, log):
         # Remove previous logs
@@ -45,7 +40,7 @@ class ScanUpstream(object):
 
         obj, created = Package.objects.get_or_create(category=cat, name=pkg)
 
-        if created and not self.options['quiet']:
+        if created and not self.quiet:
             sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
 
         # Set all versions dead, then set found versions alive and
@@ -69,7 +64,7 @@ class ScanUpstream(object):
         if not created:
             return
 
-        if not self.options['quiet']:
+        if not self.quiet:
             sys.stdout.write('+ [u] %s %s\n' % (obj, url))
 
         VersionLog.objects.create(
@@ -125,7 +120,7 @@ class ScanUpstream(object):
 
 
 @commit_on_success
-def purge_versions(options):
+def purge_versions(quiet=False):
     # For each dead versions
     for version in Version.objects.filter(packaged=False, alive=False):
         VersionLog.objects.create(
@@ -140,7 +135,7 @@ def purge_versions(options):
         version.package.n_versions -= 1
         version.package.save()
 
-        if not options['quiet']:
+        if not quiet:
             sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
     Version.objects.filter(packaged=False, alive=False).delete()
 
@@ -174,31 +169,29 @@ class Command(BaseCommand):
     help = 'Scans metadata and fills database'
 
     def handle(self, *args, **options):
-        scan_upstream = ScanUpstream(options)
+        scan_upstream = ScanUpstream(options["quiet"])
 
         if options['feed']:
-            scan_upstream.parse_output(options, sys.stdin)
+            scan_upstream.parse_output(sys.stdin)
             if options['purge-versions']:
-                purge_versions(options)
+                purge_versions(options["quiet"])
             return
 
         if not options['quiet']:
             self.stdout.write('Scanning upstream...\n')
 
-        packages = []
-
         if options['all']:
             for pkg in Package.objects.all():
-                packages.append('%s/%s' % (pkg.category, pkg.name))
+                scan_upstream.scan('%s/%s' % (pkg.category, pkg.name))
         elif args:
-            packages = list(args)
+            for arg in args:
+                scan_upstream.scan(arg)
         else:
-            packages = [package[:-1] for package in sys.stdin.readlines()]
-
-        scan_upstream.run(packages)
+            for package in sys.stdin.readlines():
+                scan_upstream.scan(package[:-1])
 
         if options['purge-versions']:
-            purge_versions(options)
+            purge_versions(options["quiet"])
 
         if not options['quiet']:
             self.stdout.write('Done.\n')
diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
new file mode 100644
index 0000000..fbc216f
--- /dev/null
+++ b/euscanwww/djeuscan/tasks.py
@@ -0,0 +1,83 @@
+from celery.task import task
+from celery.task.sets import TaskSet
+
+from djeuscan.models import Package
+
+from djeuscan.management.commands.regen_rrds import regen_rrds
+from djeuscan.management.commands.update_counters import update_counters
+from djeuscan.management.commands.scan_metadata import ScanMetadata
+from djeuscan.management.commands.scan_portage import ScanPortage, \
+    purge_versions as scan_portage_purge
+from djeuscan.management.commands.scan_upstream import ScanUpstream, \
+    purge_versions as scan_upstream_purge
+
+
+@task
+def regen_rrds_task():
+    regen_rrds()
+
+
+@task
+def update_counters_task():
+    update_counters()
+
+
+@task
+def scan_metadata_task(query, obj=None):
+    scan_metadata = ScanMetadata()
+    scan_metadata.scan(query)
+
+
+@task
+def scan_metadata_all_task():
+    job = TaskSet(tasks=[
+        scan_metadata_task.subtask(('%s/%s' % (pkg.category, pkg.name), pkg))
+        for pkg in Package.objects.all()
+    ])
+    job.apply_async()
+
+
+@task
+def scan_portage_all_task(purge=False):
+    scan_portage = ScanPortage()
+    scan_portage.scan()
+
+    if purge:
+        scan_portage_purge()
+
+
+@task
+def scan_portage_task(query, purge=False):
+    scan_portage = ScanPortage()
+    scan_portage.scan(query)
+
+    if purge:
+        scan_portage_purge()
+
+
+@task
+def scan_portage_purge_task():
+    scan_portage_purge()
+
+
+@task
+def scan_upstream_all_task(purge=False):
+    tasks = [scan_upstream_task.subtask(('%s/%s' % (pkg.category, pkg.name)))
+             for pkg in Package.objects.all()]
+
+    if purge:
+        tasks.append(scan_upstream_purge_task.subtask())
+
+    job = TaskSet(tasks=tasks)
+    job.apply_async()
+
+
+@task
+def scan_upstream_task(query):
+    scan_upstream = ScanUpstream()
+    scan_upstream.scan(query)
+
+
+@task
+def scan_upstream_purge_task():
+    scan_upstream_purge()
diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py
index 0365bc7..0f2b483 100644
--- a/euscanwww/euscanwww/settings.py
+++ b/euscanwww/euscanwww/settings.py
@@ -166,6 +166,7 @@ INSTALLED_APPS = (
     # Uncomment the next line to enable admin documentation:
     # 'django.contrib.admindocs',
     'south',
+    'djcelery',
     'euscanwww',
     'djeuscan',
 )
@@ -199,6 +200,14 @@ LOGGING = {
     }
 }
 
+
+# Celery config
+import djcelery
+djcelery.setup_loader()
+BROKER_URL = "amqp://guest:guest@localhost:5672//"
+CELERY_RESULT_BACKEND = "amqp"
+
+
 try:
     from local_settings import *
 except ImportError, ex:

From e8718748b4b499fb37516df90d093c615e9433b3 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Thu, 31 May 2012 09:11:40 +0200
Subject: [PATCH 02/37] euscanwww: Celery fixes and little logging

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/tasks.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index fbc216f..e614a70 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -24,6 +24,9 @@ def update_counters_task():
 
 @task
 def scan_metadata_task(query, obj=None):
+    logger = scan_metadata_task.get_logger()
+    logger.info("Starting metadata scanning for package %s ...", query)
+
     scan_metadata = ScanMetadata()
     scan_metadata.scan(query)
 
@@ -39,19 +42,27 @@ def scan_metadata_all_task():
 
 @task
 def scan_portage_all_task(purge=False):
+    logger = scan_portage_all_task.get_logger()
+    logger.info("Starting Portage scanning...")
+
     scan_portage = ScanPortage()
     scan_portage.scan()
 
     if purge:
+        logger.info("Purging")
         scan_portage_purge()
 
 
 @task
 def scan_portage_task(query, purge=False):
+    logger = scan_portage_task.get_logger()
+    logger.info("Starting Portage package scanning: %s ...", query)
+
     scan_portage = ScanPortage()
     scan_portage.scan(query)
 
     if purge:
+        logger.info("Purging")
         scan_portage_purge()
 
 
@@ -62,7 +73,7 @@ def scan_portage_purge_task():
 
 @task
 def scan_upstream_all_task(purge=False):
-    tasks = [scan_upstream_task.subtask(('%s/%s' % (pkg.category, pkg.name)))
+    tasks = [scan_upstream_task.subtask(('%s/%s' % (pkg.category, pkg.name), ))
              for pkg in Package.objects.all()]
 
     if purge:
@@ -74,6 +85,9 @@ def scan_upstream_all_task(purge=False):
 
 @task
 def scan_upstream_task(query):
+    logger = scan_upstream_task.get_logger()
+    logger.info("Starting upstream scanning for package %s ...", query)
+
     scan_upstream = ScanUpstream()
     scan_upstream.scan(query)
 

From 47ec539e1da48276fcd1b647414d6be52c5dda0e Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Thu, 31 May 2012 18:42:37 +0200
Subject: [PATCH 03/37] euscanwww: scan_upstream imports euscan.scan

scan_upstream command imports euscan.scan and simply calls a function,
it doesn't call a subprocess

Signed-off-by: volpino <fox91@anche.no>
---
 .../management/commands/scan_upstream.py      | 26 ++++++++++++++-----
 pym/euscan/__init__.py                        |  2 +-
 pym/euscan/out.py                             |  9 ++++---
 3 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/euscanwww/djeuscan/management/commands/scan_upstream.py b/euscanwww/djeuscan/management/commands/scan_upstream.py
index 2e2d9c2..131ff1a 100644
--- a/euscanwww/djeuscan/management/commands/scan_upstream.py
+++ b/euscanwww/djeuscan/management/commands/scan_upstream.py
@@ -1,14 +1,16 @@
-import subprocess
 import portage
 import sys
 import re
-from StringIO import StringIO
+
 from optparse import make_option
 
 from django.utils import timezone
 from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand
 
+from euscan import CONFIG, output
+from euscan.scan import scan_upstream
+
 from djeuscan.models import Package, Version, EuscanResult, VersionLog
 
 
@@ -17,13 +19,23 @@ class ScanUpstream(object):
         self.quiet = quiet
 
     def scan(self, package):
-        cmd = ['euscan', package]
+        CONFIG["format"] = "dict"
+        output.set_query(package)
+        scan_upstream(package)
+        out = output.get_formatted_output()
+        out_json = output.get_formatted_output("json")
 
-        fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE)
-        output = StringIO(fp.communicate()[0])
+        try:
+            cp = out["metadata"]["cp"]
+        except KeyError:
+            return {}
 
-        self.parse_output(output)
+        for out in out["result"]:
+            self.store_version(cp, out["version"], " ".join(out["urls"]))
+
+        self.store_result(cp, out_json)
+
+        return out
 
     def store_result(self, package, log):
         # Remove previous logs
diff --git a/pym/euscan/__init__.py b/pym/euscan/__init__.py
index 01255f4..84754ee 100644
--- a/pym/euscan/__init__.py
+++ b/pym/euscan/__init__.py
@@ -67,4 +67,4 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [
 ]
 
 from out import EuscanOutput
-output = out.EuscanOutput(CONFIG)
+output = EuscanOutput(CONFIG)
diff --git a/pym/euscan/out.py b/pym/euscan/out.py
index c569a89..6a98336 100644
--- a/pym/euscan/out.py
+++ b/pym/euscan/out.py
@@ -95,7 +95,7 @@ class EuscanOutput(object):
             "metadata": {},
         }
 
-    def get_formatted_output(self):
+    def get_formatted_output(self, format_=None):
         data = {}
 
         for query in self.queries:
@@ -105,8 +105,11 @@ class EuscanOutput(object):
                 "messages": self.queries[query]["output"].getvalue(),
             }
 
-        if self.config["format"].lower() == "json":
+        format_ = format_ or self.config["format"]
+        if format_.lower() == "json":
             return json.dumps(data, indent=self.config["indent"])
+        elif format_.lower() == "dict":
+            return data
         else:
             raise TypeError("Invalid output format")
 
@@ -133,7 +136,7 @@ class EuscanOutput(object):
             print "%s: %s" % (key.capitalize(), value)
 
     def __getattr__(self, key):
-        if not self.config["quiet"]:
+        if not self.config["quiet"] and self.current_query is not None:
             output = self.queries[self.current_query]["output"]
             return getattr(output, key)
         else:

From 5970b4df3727e90053dfb9470570010e756ca069 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Sun, 3 Jun 2012 09:43:49 +0200
Subject: [PATCH 04/37] euscanwww: Admin improved and simple task launcher
 added

Signed-off-by: volpino <fox91@anche.no>
---
 .../management/commands/scan_metadata.py      |  2 +-
 .../management/commands/scan_portage.py       |  8 +-
 euscanwww/djeuscan/models.py                  |  5 ++
 euscanwww/djeuscan/tasks.py                   | 16 +++-
 .../admin/djcelery/taskstate/change_list.html | 77 +++++++++++++++++++
 euscanwww/djeuscan/urls.py                    | 17 ++++
 euscanwww/djeuscan/views.py                   | 22 +++++-
 euscanwww/euscanwww/settings.py               |  5 +-
 8 files changed, 143 insertions(+), 9 deletions(-)
 create mode 100644 euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html

diff --git a/euscanwww/djeuscan/management/commands/scan_metadata.py b/euscanwww/djeuscan/management/commands/scan_metadata.py
index d9c7e49..c3f4c4b 100644
--- a/euscanwww/djeuscan/management/commands/scan_metadata.py
+++ b/euscanwww/djeuscan/management/commands/scan_metadata.py
@@ -13,7 +13,7 @@ from djeuscan.models import Package, Herd, Maintainer
 
 
 class ScanMetadata(object):
-    def __init__(self, quiet):
+    def __init__(self, quiet=False):
         self.quiet = quiet
         self.style = color_style()
 
diff --git a/euscanwww/djeuscan/management/commands/scan_portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py
index 9533730..4471fac 100644
--- a/euscanwww/djeuscan/management/commands/scan_portage.py
+++ b/euscanwww/djeuscan/management/commands/scan_portage.py
@@ -241,7 +241,7 @@ class ScanPortage(object):
 
 
 @commit_on_success
-def purge_versions(options):
+def purge_versions(quiet=False, nolog=False):
     # For each dead versions
     for version in Version.objects.filter(packaged=True, alive=False):
         if version.overlay == 'gentoo':
@@ -251,10 +251,10 @@ def purge_versions(options):
         version.package.n_versions -= 1
         version.package.save()
 
-        if not options['quiet']:
+        if not quiet:
             sys.stdout.write('- [v] %s\n' % (version))
 
-        if options['no-log']:
+        if nolog:
             continue
 
         VersionLog.objects.create(
@@ -335,7 +335,7 @@ class Command(BaseCommand):
                 scan_portage.scan(package[:-1])
 
         if options['purge-versions']:
-            purge_versions(options)
+            purge_versions(options["quiet"], options["no-log"])
 
         if not options['quiet']:
             self.stdout.write('Done.\n')
diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py
index 727deef..fee8dc4 100644
--- a/euscanwww/djeuscan/models.py
+++ b/euscanwww/djeuscan/models.py
@@ -190,6 +190,11 @@ class EuscanResult(models.Model):
         self.full_clean()
         super(EuscanResult, self).save(*args, **kwargs)
 
+    def __unicode__(self):
+        return '[%s] %s/%s' % (
+            self.datetime, self.package.category, self.package.name
+        )
+
 
 class Log(models.Model):
     """
diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index e614a70..89a90dc 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -28,7 +28,7 @@ def scan_metadata_task(query, obj=None):
     logger.info("Starting metadata scanning for package %s ...", query)
 
     scan_metadata = ScanMetadata()
-    scan_metadata.scan(query)
+    scan_metadata.scan(query, obj)
 
 
 @task
@@ -95,3 +95,17 @@ def scan_upstream_task(query):
 @task
 def scan_upstream_purge_task():
     scan_upstream_purge()
+
+
+launchable_tasks = [
+    regen_rrds_task,
+    update_counters_task,
+    scan_metadata_task,
+    scan_metadata_all_task,
+    scan_portage_all_task,
+    scan_portage_task,
+    scan_portage_purge_task,
+    scan_upstream_all_task,
+    scan_upstream_task,
+    scan_upstream_purge_task,
+]
diff --git a/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html b/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html
new file mode 100644
index 0000000..693214a
--- /dev/null
+++ b/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html
@@ -0,0 +1,77 @@
+{% extends "admin/change_list.html" %}
+
+{% load url from future %}
+
+{% block object-tools %}
+  <div>
+    <span id="task_selector"></span>
+    <form id="task_data"></form>
+    <a href="#" id="task_launch">Launch task</a>
+  </div>
+
+  <script type="text/javascript">
+    (function($){
+      $(document).ready(function() {
+
+        $.get("{% url "registered_tasks" %}", function (data) {
+          var selector = $("<select/>");
+          selector.append(
+            $("<option/>").text("-------------").attr("selected", "selected")
+          );
+          for (task in data["tasks"]) {
+            var t = data["tasks"][task];
+            selector.append(
+              $("<option/>").val(task).text(task)
+            )
+          }
+
+          selector.change(function() {
+            $("#task_data").empty();
+            var task = data["tasks"][$(this).val()];
+            if (task.args) {
+              for (var i = 0; i < task.args.length; i++) {
+                var arg = task.args[i];
+                var arg_input = $("<input/>").attr("name", arg);
+                var default_arg;
+
+                if (task.defaults) {
+                  var default_arg_i = task.args.length - 1 - i;
+                  if (task.defaults[default_arg_i] !== undefined) {
+                    default_arg = task.defaults[default_arg_i];
+                    if (task.defaults_types && task.defaults_types[default_arg_i] === "bool") {
+                      arg_input.attr("type", "checkbox");
+                    }
+                    else {
+                      continue
+                    }
+                  }
+                }
+
+                arg_input.val(default_arg);
+
+                $("#task_data").append(
+                  $("<label/>").text(arg + ": ").attr("for", arg)
+                ).append(
+                  arg_input
+                );
+              }
+            }
+          })
+
+          $("#task_selector").append(selector);
+        });
+
+        $("#task_launch").click(function() {
+            var task_name = $("#task_selector").find("select").val();
+            var data = $("#task_data").serialize();
+            var url = "{% url "apply_task" "task_name" %}";
+
+            $.post(url.replace("task_name", task_name), data, function() {
+              location.reload();
+            });
+        });
+
+      });
+    })(django.jQuery);
+  </script>
+{% endblock %}
diff --git a/euscanwww/djeuscan/urls.py b/euscanwww/djeuscan/urls.py
index 019f99a..721fad5 100644
--- a/euscanwww/djeuscan/urls.py
+++ b/euscanwww/djeuscan/urls.py
@@ -1,7 +1,16 @@
 from django.conf.urls.defaults import url, patterns, include
+from django.contrib.auth.decorators import user_passes_test
+
+from djcelery.views import apply
+from djeuscan.views import registered_tasks
+
 from djeuscan.feeds import PackageFeed, CategoryFeed, HerdFeed, \
     MaintainerFeed, GlobalFeed
 
+
+admin_required = user_passes_test(lambda u: u.is_superuser)
+
+
 package_patterns = patterns('djeuscan.views',
     url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/feed/$',
         PackageFeed(), name='package_feed'),
@@ -41,6 +50,13 @@ overlays_patterns = patterns('djeuscan.views',
     url(r'^$', 'overlays', name="overlays"),
 )
 
+tasks_patterns = patterns('djcelery.views',
+    url(r'^registered_tasks/$', admin_required(registered_tasks),
+        name="registered_tasks"),
+    url(r'^apply/(?P<task_name>.*)/$', admin_required(apply),
+        name="apply_task"),
+)
+
 urlpatterns = patterns('djeuscan.views',
     # Global stuff
     url(r'^api/', include('djeuscan.api.urls')),
@@ -60,4 +76,5 @@ urlpatterns = patterns('djeuscan.views',
     url(r'^maintainers/', include(maintainers_patterns)),
     url(r'^overlays/', include(overlays_patterns)),
     url(r'^package/', include(package_patterns)),
+    url(r'^tasks/', include(tasks_patterns)),
 )
diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py
index 68836af..d84c39d 100644
--- a/euscanwww/djeuscan/views.py
+++ b/euscanwww/djeuscan/views.py
@@ -1,6 +1,8 @@
 """ Views """
 
-from annoying.decorators import render_to
+import inspect
+from annoying.decorators import render_to, ajax_request
+
 from django.http import Http404
 from django.shortcuts import get_object_or_404
 
@@ -8,6 +10,7 @@ from djeuscan.helpers import version_key, packages_from_names
 from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
     VersionLog
 from djeuscan.forms import WorldForm, PackagesForm
+from djeuscan.tasks import launchable_tasks
 from djeuscan import charts
 
 
@@ -254,3 +257,20 @@ def chart_herd(request, **kwargs):
 
 def chart_category(request, **kwargs):
     return chart(request, **kwargs)
+
+
+@ajax_request
+def registered_tasks(request):
+    data = {}
+    for task in launchable_tasks:
+        argspec = inspect.getargspec(task.run)
+        data[task.name] = {
+            "args": argspec.args,
+            "defaults": argspec.defaults,
+            "default_types": None
+        }
+        if argspec.defaults is not None:
+            data[task.name].update({
+                "defaults_types": [type(x).__name__ for x in argspec.defaults]
+            })
+    return {"tasks": data}
diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py
index 0f2b483..d84e340 100644
--- a/euscanwww/euscanwww/settings.py
+++ b/euscanwww/euscanwww/settings.py
@@ -157,6 +157,9 @@ TEMPLATE_CONTEXT_PROCESSORS = (
 )
 
 INSTALLED_APPS = (
+    'euscanwww',
+    'djeuscan',
+
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
@@ -167,8 +170,6 @@ INSTALLED_APPS = (
     # 'django.contrib.admindocs',
     'south',
     'djcelery',
-    'euscanwww',
-    'djeuscan',
 )
 
 # A sample logging configuration. The only tangible logging

From 2edb220d3edf43b7e48d1d0152bd91f32eaa5edb Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Sun, 3 Jun 2012 17:38:51 +0200
Subject: [PATCH 05/37] euscanwww: use jQuery each instead of for

Signed-off-by: volpino <fox91@anche.no>
---
 .../admin/djcelery/taskstate/change_list.html | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html b/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html
index 693214a..a4780cd 100644
--- a/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html
+++ b/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html
@@ -18,21 +18,22 @@
           selector.append(
             $("<option/>").text("-------------").attr("selected", "selected")
           );
-          for (task in data["tasks"]) {
+          $.each(data.tasks, function(task) {
             var t = data["tasks"][task];
             selector.append(
               $("<option/>").val(task).text(task)
             )
-          }
+          });
 
           selector.change(function() {
-            $("#task_data").empty();
             var task = data["tasks"][$(this).val()];
+            $("#task_data").empty();
+
             if (task.args) {
-              for (var i = 0; i < task.args.length; i++) {
-                var arg = task.args[i];
-                var arg_input = $("<input/>").attr("name", arg);
-                var default_arg;
+              $.each(task.args, function(i) {
+                var arg = task.args[i]
+                  , arg_input = $("<input/>").attr("name", arg)
+                  ,  default_arg;
 
                 if (task.defaults) {
                   var default_arg_i = task.args.length - 1 - i;
@@ -42,7 +43,7 @@
                       arg_input.attr("type", "checkbox");
                     }
                     else {
-                      continue
+                      return true;
                     }
                   }
                 }
@@ -54,7 +55,7 @@
                 ).append(
                   arg_input
                 );
-              }
+              });
             }
           })
 

From 60e2019cc3472f8e8d9065b2e52bbceb1234bfaf Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Mon, 4 Jun 2012 11:06:47 +0200
Subject: [PATCH 06/37] euscanwww: Added tasks for syncing and updating portage
 tree

Added these tasks:
 * emerge_sync
 * layman_sync
 * emerge_regen
 * eix_update

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/tasks.py     | 45 ++++++++++++++++++++++++++++++++-
 euscanwww/euscanwww/settings.py | 14 +++++++---
 euscanwww/scripts/euscan.fcgi   |  3 ++-
 3 files changed, 56 insertions(+), 6 deletions(-)

diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index 89a90dc..2f36f95 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -1,8 +1,12 @@
+import subprocess
+from StringIO import StringIO
+
 from celery.task import task
 from celery.task.sets import TaskSet
 
-from djeuscan.models import Package
+from django.conf import settings
 
+from djeuscan.models import Package
 from djeuscan.management.commands.regen_rrds import regen_rrds
 from djeuscan.management.commands.update_counters import update_counters
 from djeuscan.management.commands.scan_metadata import ScanMetadata
@@ -97,6 +101,41 @@ def scan_upstream_purge_task():
     scan_upstream_purge()
 
 
+def _launch_command(cmd):
+    fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+    output = StringIO(fp.communicate()[0])
+    return output.getvalue()
+
+
+@task
+def emerge_sync():
+    cmd = ["emerge", "--sync", "--root", settings.PORTAGE_ROOT,
+           "--config-root", settings.PORTAGE_CONFIGROOT]
+    return _launch_command(cmd)
+
+
+@task
+def layman_sync():
+    cmd = ["layman", "-S", "--config", settings.LAYMAN_CONFIG]
+    return _launch_command(cmd)
+
+
+@task
+def emerge_regen():
+    cmd = [
+        "emerge", "--regen", "--jobs", settings.EMERGE_REGEN_JOBS, "--root",
+        settings.PORTAGE_ROOT, "--config-root", settings.PORTAGE_CONFIGROOT
+    ]
+    return _launch_command(cmd)
+
+
+@task
+def eix_update():
+    cmd = ["eix-update"]
+    return _launch_command(cmd)
+
+
 launchable_tasks = [
     regen_rrds_task,
     update_counters_task,
@@ -108,4 +147,8 @@ launchable_tasks = [
     scan_upstream_all_task,
     scan_upstream_task,
     scan_upstream_purge_task,
+    emerge_sync,
+    layman_sync,
+    emerge_regen,
+    eix_update,
 ]
diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py
index d84e340..46182c0 100644
--- a/euscanwww/euscanwww/settings.py
+++ b/euscanwww/euscanwww/settings.py
@@ -202,6 +202,12 @@ LOGGING = {
 }
 
 
+PORTAGE_ROOT = "/usr/portage/"
+PORTAGE_CONFIGROOT = PORTAGE_ROOT
+LAYMAN_CONFIG = "/etc/layman/layman.cfg"
+EMERGE_REGEN_JOBS = 4
+
+
 # Celery config
 import djcelery
 djcelery.setup_loader()
@@ -213,7 +219,7 @@ try:
     from local_settings import *
 except ImportError, ex:
     import sys
-    sys.stderr.write(\
-            ("settings.py: error importing local settings file:\n" + \
-            "\t%s\n" + \
-            "Do you have a local_settings.py module?\n") % str(ex))
+    sys.stderr.write(
+        "settings.py: error importing local settings file:\n"
+        "\t%s\nDo you have a local_settings.py module?\n" % str(ex)
+    )
diff --git a/euscanwww/scripts/euscan.fcgi b/euscanwww/scripts/euscan.fcgi
index d9d5b91..166bbdb 100755
--- a/euscanwww/scripts/euscan.fcgi
+++ b/euscanwww/scripts/euscan.fcgi
@@ -1,6 +1,7 @@
 #!/usr/bin/python
 
-import sys, os
+import sys
+import os
 import os.path
 
 PROJECT = '/path/to/euscanwww'

From c41521d59f4b7b34e0603b0f200b568e6a1ddd72 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Mon, 4 Jun 2012 11:27:18 +0200
Subject: [PATCH 07/37] euscanwww: Admin interface tweaks

Added some search fields, ordering and filters to the admin panel

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/admin.py | 32 ++++++++++++++++++++++++++++----
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/euscanwww/djeuscan/admin.py b/euscanwww/djeuscan/admin.py
index 45bdbeb..329117a 100644
--- a/euscanwww/djeuscan/admin.py
+++ b/euscanwww/djeuscan/admin.py
@@ -2,18 +2,42 @@ from djeuscan.models import Package, Version, VersionLog, EuscanResult, \
     Log, WorldLog, CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer
 from django.contrib import admin
 
-admin.site.register(Herd)
-admin.site.register(Maintainer)
+
+class EuscanResultAdmin(admin.ModelAdmin):
+    search_fields = ('package__name', 'package__category')
+    list_filter = ('datetime', )
+    ordering = ["-datetime"]
+
+
+class HerdAdmin(admin.ModelAdmin):
+    search_fields = ('herd', 'email')
+    ordering = ["herd"]
+
+
+class MaintainerAdmin(admin.ModelAdmin):
+    search_fields = ('name', 'email')
+    ordering = ["name"]
 
 
 class PackageAdmin(admin.ModelAdmin):
     search_fields = ('category', 'name')
+    list_filter = ('category', )
+
+
+class VersionAdmin(admin.ModelAdmin):
+    search_fields = ('package__name', 'package__category')
+    list_filter = ('overlay', 'packaged', 'alive')
+
 
 admin.site.register(Package, PackageAdmin)
 
-admin.site.register(Version)
+admin.site.register(Herd, HerdAdmin)
+admin.site.register(Maintainer, MaintainerAdmin)
+
+admin.site.register(Version, VersionAdmin)
 admin.site.register(VersionLog)
-admin.site.register(EuscanResult)
+
+admin.site.register(EuscanResult, EuscanResultAdmin)
 
 admin.site.register(Log)
 admin.site.register(WorldLog)

From 8ed74a74dfe1484d402146b391e769347ab56cbd Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Tue, 5 Jun 2012 12:46:22 +0200
Subject: [PATCH 08/37] euscanwww: Changed Layman sync task to use Python API

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/tasks.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index 2f36f95..a112a26 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -117,8 +117,12 @@ def emerge_sync():
 
 @task
 def layman_sync():
-    cmd = ["layman", "-S", "--config", settings.LAYMAN_CONFIG]
-    return _launch_command(cmd)
+    from layman import Layman
+    results = []
+    l = Layman()
+    for overlay in l.get_installed():
+        results.append(l.sync(overlay, output_results=False))
+    return results
 
 
 @task

From 4a9301a215ee3e3f36d2cecfc14a9a4976b38079 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Tue, 5 Jun 2012 13:10:25 +0200
Subject: [PATCH 09/37] euscanwww: naive implementation of the refresh button

A refresh button lets launching a scan_upstream task for a package while
navigating into euscanwww

Signed-off-by: volpino <fox91@anche.no>
---
 .../djeuscan/templates/euscan/package.html    | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/euscanwww/djeuscan/templates/euscan/package.html b/euscanwww/djeuscan/templates/euscan/package.html
index c5af507..5b3fb81 100644
--- a/euscanwww/djeuscan/templates/euscan/package.html
+++ b/euscanwww/djeuscan/templates/euscan/package.html
@@ -24,7 +24,15 @@
 {% endblock %}
 
 {% block content %}
-<h2>{{ package.category }}/{{ package.name }}</h2>
+<h2>
+  {{ package.category }}/{{ package.name }}
+
+  {% if user.is_superuser %}
+    <button class="refresh-button" data-package="{{ package.category }}/{{ package.name }}">
+      Refresh
+    </button>
+  {% endif %}
+</h2>
 <dl>
   {% if package.description %}
   <dt>Description</dt>
@@ -129,4 +137,14 @@
   </dd>
   {% endif %}
 </dl>
+
+<script type="text/javascript">
+  $(".refresh-button").click(function() {
+    var url = "{% url "apply_task" "djeuscan.tasks.scan_upstream_task" %}";
+    $.post(url, {query: $(this).data("package")}, function() {
+      alert("Submitted!");
+    });
+  });
+</script>
+
 {% endblock %}

From 8501805f8e2d99d8ec8615f05806db9e5c447a9b Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Tue, 5 Jun 2012 13:30:31 +0200
Subject: [PATCH 10/37] euscanwww: Scan tasks recieve a list of packages

Signed-off-by: volpino <fox91@anche.no>
---
 .../management/commands/scan_portage.py       |  3 +-
 euscanwww/djeuscan/tasks.py                   | 68 ++++++++++++-------
 2 files changed, 46 insertions(+), 25 deletions(-)

diff --git a/euscanwww/djeuscan/management/commands/scan_portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py
index 4471fac..6a1cbef 100644
--- a/euscanwww/djeuscan/management/commands/scan_portage.py
+++ b/euscanwww/djeuscan/management/commands/scan_portage.py
@@ -16,7 +16,8 @@ from djeuscan.models import Package, Version, VersionLog
 class ScanPortage(object):
     def __init__(self, stdout=None, options=None):
         self.stdout = sys.stdout if stdout is None else stdout
-        self.options = defaultdict(None) if options is None else options
+        self.options = defaultdict(lambda: None) \
+                       if options is None else options
 
         self.style = color_style()
         self._cache = {'packages': {}, 'versions': {}}
diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index a112a26..64f3ea1 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -35,6 +35,15 @@ def scan_metadata_task(query, obj=None):
     scan_metadata.scan(query, obj)
 
 
+@task
+def scan_metadata_list_task(query):
+    job = TaskSet(tasks=[
+        scan_metadata_task.subtask((pkg, ))
+        for pkg in query.split()
+    ])
+    job.apply_async()
+
+
 @task
 def scan_metadata_all_task():
     job = TaskSet(tasks=[
@@ -44,6 +53,21 @@ def scan_metadata_all_task():
     job.apply_async()
 
 
+@task
+def scan_portage_list_task(query, purge=False):
+    scan_portage = ScanPortage()
+
+    for pkg in query.split():
+        logger = scan_portage_list_task.get_logger()
+        logger.info("Starting Portage package scanning: %s ...", pkg)
+
+        scan_portage.scan(pkg)
+
+    if purge:
+        logger.info("Purging")
+        scan_portage_purge()
+
+
 @task
 def scan_portage_all_task(purge=False):
     logger = scan_portage_all_task.get_logger()
@@ -58,21 +82,26 @@ def scan_portage_all_task(purge=False):
 
 
 @task
-def scan_portage_task(query, purge=False):
-    logger = scan_portage_task.get_logger()
-    logger.info("Starting Portage package scanning: %s ...", query)
-
-    scan_portage = ScanPortage()
-    scan_portage.scan(query)
-
-    if purge:
-        logger.info("Purging")
-        scan_portage_purge()
+def scan_portage_purge_task():
+    scan_portage_purge()
 
 
 @task
-def scan_portage_purge_task():
-    scan_portage_purge()
+def scan_upstream_task(query):
+    logger = scan_upstream_task.get_logger()
+    logger.info("Starting upstream scanning for package %s ...", query)
+
+    scan_upstream = ScanUpstream()
+    scan_upstream.scan(query)
+
+
+@task
+def scan_upstream_list_task(query):
+    job = TaskSet(tasks=[
+        scan_upstream_task.subtask((pkg, ))
+        for pkg in query.split()
+    ])
+    job.apply_async()
 
 
 @task
@@ -87,15 +116,6 @@ def scan_upstream_all_task(purge=False):
     job.apply_async()
 
 
-@task
-def scan_upstream_task(query):
-    logger = scan_upstream_task.get_logger()
-    logger.info("Starting upstream scanning for package %s ...", query)
-
-    scan_upstream = ScanUpstream()
-    scan_upstream.scan(query)
-
-
 @task
 def scan_upstream_purge_task():
     scan_upstream_purge()
@@ -143,13 +163,13 @@ def eix_update():
 launchable_tasks = [
     regen_rrds_task,
     update_counters_task,
-    scan_metadata_task,
+    scan_metadata_list_task,
     scan_metadata_all_task,
     scan_portage_all_task,
-    scan_portage_task,
+    scan_portage_list_task,
     scan_portage_purge_task,
     scan_upstream_all_task,
-    scan_upstream_task,
+    scan_upstream_list_task,
     scan_upstream_purge_task,
     emerge_sync,
     layman_sync,

From 33555a9fbe41e214a342798a94dfe4e87a2cb19f Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Tue, 5 Jun 2012 16:43:40 +0200
Subject: [PATCH 11/37] euscanwww: Fixed Layman sync task

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/tasks.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index 64f3ea1..f29f69a 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -138,11 +138,8 @@ def emerge_sync():
 @task
 def layman_sync():
     from layman import Layman
-    results = []
-    l = Layman()
-    for overlay in l.get_installed():
-        results.append(l.sync(overlay, output_results=False))
-    return results
+    l = Layman(config=settings.LAYMAN_CONFIG)
+    return l.sync(l.get_installed(), output_results=False)
 
 
 @task

From 8913f826de0dd4e461ed8b36ef8654438935b38b Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Fri, 8 Jun 2012 14:19:20 +0200
Subject: [PATCH 12/37] euscanwww: Added RefreshPackageQuery to models

Allows storing queries for refresh requests by users

Signed-off-by: volpino <fox91@anche.no>
---
 ...chg_field_package_last_version_gentoo__.py | 139 ++++++++++++++++++
 euscanwww/djeuscan/models.py                  |  19 ++-
 2 files changed, 154 insertions(+), 4 deletions(-)
 create mode 100644 euscanwww/djeuscan/migrations/0003_auto__add_refreshpackagequery__chg_field_package_last_version_gentoo__.py

diff --git a/euscanwww/djeuscan/migrations/0003_auto__add_refreshpackagequery__chg_field_package_last_version_gentoo__.py b/euscanwww/djeuscan/migrations/0003_auto__add_refreshpackagequery__chg_field_package_last_version_gentoo__.py
new file mode 100644
index 0000000..2724c09
--- /dev/null
+++ b/euscanwww/djeuscan/migrations/0003_auto__add_refreshpackagequery__chg_field_package_last_version_gentoo__.py
@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'RefreshPackageQuery'
+        db.create_table('djeuscan_refreshpackagequery', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('query', self.gf('django.db.models.fields.CharField')(unique=True, max_length=256)),
+            ('priority', self.gf('django.db.models.fields.IntegerField')(default=0)),
+        ))
+        db.send_create_signal('djeuscan', ['RefreshPackageQuery'])
+
+
+        # Changing field 'Package.last_version_gentoo'
+        db.alter_column('djeuscan_package', 'last_version_gentoo_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['djeuscan.Version']))
+
+        # Changing field 'Package.last_version_overlay'
+        db.alter_column('djeuscan_package', 'last_version_overlay_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['djeuscan.Version']))
+
+        # Changing field 'Package.last_version_upstream'
+        db.alter_column('djeuscan_package', 'last_version_upstream_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['djeuscan.Version']))
+    def backwards(self, orm):
+        # Deleting model 'RefreshPackageQuery'
+        db.delete_table('djeuscan_refreshpackagequery')
+
+
+        # Changing field 'Package.last_version_gentoo'
+        db.alter_column('djeuscan_package', 'last_version_gentoo_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['djeuscan.Version']))
+
+        # Changing field 'Package.last_version_overlay'
+        db.alter_column('djeuscan_package', 'last_version_overlay_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['djeuscan.Version']))
+
+        # Changing field 'Package.last_version_upstream'
+        db.alter_column('djeuscan_package', 'last_version_upstream_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['djeuscan.Version']))
+    models = {
+        'djeuscan.categorylog': {
+            'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.euscanresult': {
+            'Meta': {'object_name': 'EuscanResult'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'djeuscan.herd': {
+            'Meta': {'object_name': 'Herd'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
+            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'djeuscan.herdlog': {
+            'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.log': {
+            'Meta': {'object_name': 'Log'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'djeuscan.maintainer': {
+            'Meta': {'object_name': 'Maintainer'},
+            'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.maintainerlog': {
+            'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
+        },
+        'djeuscan.package': {
+            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
+            'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.refreshpackagequery': {
+            'Meta': {'object_name': 'RefreshPackageQuery'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
+        },
+        'djeuscan.version': {
+            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
+            'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.versionlog': {
+            'Meta': {'object_name': 'VersionLog'},
+            'action': ('django.db.models.fields.IntegerField', [], {}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.worldlog': {
+            'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        }
+    }
+
+    complete_apps = ['djeuscan']
\ No newline at end of file
diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py
index fee8dc4..162e4ef 100644
--- a/euscanwww/djeuscan/models.py
+++ b/euscanwww/djeuscan/models.py
@@ -115,12 +115,12 @@ class Version(models.Model):
     """
 
     package = models.ForeignKey(Package)
-    slot = models.CharField(max_length=128)
+    slot = models.CharField(max_length=128, blank=True)
     revision = models.CharField(max_length=128)
     version = models.CharField(max_length=128)
     packaged = models.BooleanField()
     overlay = models.CharField(max_length=128, default='gentoo', db_index=True,
-                               validators=[validate_name])
+                               validators=[validate_name], blank=True)
     urls = models.TextField(blank=True)
     alive = models.BooleanField(default=True, db_index=True)
 
@@ -148,12 +148,12 @@ class VersionLog(models.Model):
 
     package = models.ForeignKey(Package)
     datetime = models.DateTimeField(auto_now_add=True)
-    slot = models.CharField(max_length=128)
+    slot = models.CharField(max_length=128, blank=True)
     revision = models.CharField(max_length=128)
     version = models.CharField(max_length=128)
     packaged = models.BooleanField()
     overlay = models.CharField(max_length=128, default='gentoo',
-                               validators=[validate_name])
+                               validators=[validate_name], blank=True)
     action = models.IntegerField(choices=VERSION_ACTIONS)
 
     objects = VersionLogManager()
@@ -257,3 +257,14 @@ class MaintainerLog(Log):
 
     def __unicode__(self):
         return u'%s %s' % (self.maintainer, Log.__unicode__(self))
+
+
+class RefreshPackageQuery(models.Model):
+    query = models.CharField(max_length=256, unique=True)
+    priority = models.IntegerField(default=0)
+
+    class Meta:
+        get_latest_by = "priority"
+
+    def __unicode__(self):
+        return u'[%d] %s' % (self.priority, self.query)

From a94a24eeac582dcb817b2599254bdf409606b155 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Fri, 8 Jun 2012 14:21:11 +0200
Subject: [PATCH 13/37] euscanwww: Refresh query logic and task queue split in
 chunks

Added the needed logic for running refresh queries from the web
interface.

Added a simple function to tasks.py that takes an iterable of arguments
and a task and launches many subtasks formed by 32 tasks each (for not
having a huge amount of tasks launched at the same time)

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/tasks.py                   | 95 ++++++++++++-------
 .../djeuscan/templates/euscan/package.html    |  4 +-
 euscanwww/djeuscan/urls.py                    |  5 +-
 euscanwww/djeuscan/views.py                   | 19 +++-
 4 files changed, 82 insertions(+), 41 deletions(-)

diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index f29f69a..77064b0 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -1,12 +1,14 @@
 import subprocess
 from StringIO import StringIO
+from itertools import islice
 
-from celery.task import task
+from celery.task import task, periodic_task
+from celery.task.schedules import crontab
 from celery.task.sets import TaskSet
 
 from django.conf import settings
 
-from djeuscan.models import Package
+from djeuscan.models import Package, RefreshPackageQuery
 from djeuscan.management.commands.regen_rrds import regen_rrds
 from djeuscan.management.commands.update_counters import update_counters
 from djeuscan.management.commands.scan_metadata import ScanMetadata
@@ -16,14 +18,38 @@ from djeuscan.management.commands.scan_upstream import ScanUpstream, \
     purge_versions as scan_upstream_purge
 
 
+def _launch_command(cmd):
+    fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+    output = StringIO(fp.communicate()[0])
+    return output.getvalue()
+
+
+def _chunks(it, n):
+    for first in it:
+        yield [first] + list(islice(it, n - 1))
+
+
+def _run_in_chunks(task, iterable, n=32):
+    output = []
+    for chunk in _chunks(iter(iterable), n):
+        job = TaskSet(tasks=[
+            task.subtask(args)
+            for args in chunk
+        ])
+        result = job.apply_async()
+        output.extend(list(result.join()))
+    return output
+
+
 @task
 def regen_rrds_task():
-    regen_rrds()
+    return regen_rrds()
 
 
 @task
 def update_counters_task():
-    update_counters()
+    return update_counters()
 
 
 @task
@@ -32,33 +58,29 @@ def scan_metadata_task(query, obj=None):
     logger.info("Starting metadata scanning for package %s ...", query)
 
     scan_metadata = ScanMetadata()
-    scan_metadata.scan(query, obj)
+    return scan_metadata.scan(query, obj)
 
 
 @task
 def scan_metadata_list_task(query):
-    job = TaskSet(tasks=[
-        scan_metadata_task.subtask((pkg, ))
-        for pkg in query.split()
-    ])
-    job.apply_async()
+    return _run_in_chunks(scan_metadata_task, [(p, ) for p in query.split()])
 
 
 @task
 def scan_metadata_all_task():
-    job = TaskSet(tasks=[
-        scan_metadata_task.subtask(('%s/%s' % (pkg.category, pkg.name), pkg))
-        for pkg in Package.objects.all()
-    ])
-    job.apply_async()
+    return _run_in_chunks(
+        scan_metadata_task,
+        [('%s/%s' % (pkg.category, pkg.name), pkg)
+         for pkg in Package.objects.all()]
+    )
 
 
 @task
 def scan_portage_list_task(query, purge=False):
     scan_portage = ScanPortage()
+    logger = scan_portage_list_task.get_logger()
 
     for pkg in query.split():
-        logger = scan_portage_list_task.get_logger()
         logger.info("Starting Portage package scanning: %s ...", pkg)
 
         scan_portage.scan(pkg)
@@ -92,28 +114,26 @@ def scan_upstream_task(query):
     logger.info("Starting upstream scanning for package %s ...", query)
 
     scan_upstream = ScanUpstream()
-    scan_upstream.scan(query)
+    return scan_upstream.scan(query)
 
 
 @task
 def scan_upstream_list_task(query):
-    job = TaskSet(tasks=[
-        scan_upstream_task.subtask((pkg, ))
-        for pkg in query.split()
-    ])
-    job.apply_async()
+    return _run_in_chunks(scan_upstream_task, [(p, ) for p in query.split()])
 
 
 @task
 def scan_upstream_all_task(purge=False):
-    tasks = [scan_upstream_task.subtask(('%s/%s' % (pkg.category, pkg.name), ))
-             for pkg in Package.objects.all()]
+    output = _run_in_chunks(
+        scan_upstream_task,
+        [('%s/%s' % (pkg.category, pkg.name), )
+         for pkg in Package.objects.all()]
+    )
 
     if purge:
-        tasks.append(scan_upstream_purge_task.subtask())
+        output += [scan_upstream_purge_task()]
 
-    job = TaskSet(tasks=tasks)
-    job.apply_async()
+    return output
 
 
 @task
@@ -121,13 +141,6 @@ def scan_upstream_purge_task():
     scan_upstream_purge()
 
 
-def _launch_command(cmd):
-    fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                          stderr=subprocess.PIPE)
-    output = StringIO(fp.communicate()[0])
-    return output.getvalue()
-
-
 @task
 def emerge_sync():
     cmd = ["emerge", "--sync", "--root", settings.PORTAGE_ROOT,
@@ -157,7 +170,19 @@ def eix_update():
     return _launch_command(cmd)
 
 
-launchable_tasks = [
+@periodic_task(run_every=crontab(minute="*/1"))
+def refresh_package_consume():
+    try:
+        obj = RefreshPackageQuery.objects.latest()
+    except RefreshPackageQuery.DoesNotExist:
+        return {}
+    else:
+        result = scan_upstream_task(obj.query)
+        obj.delete()
+        return result
+
+
+admin_tasks = [
     regen_rrds_task,
     update_counters_task,
     scan_metadata_list_task,
diff --git a/euscanwww/djeuscan/templates/euscan/package.html b/euscanwww/djeuscan/templates/euscan/package.html
index 5b3fb81..f042d04 100644
--- a/euscanwww/djeuscan/templates/euscan/package.html
+++ b/euscanwww/djeuscan/templates/euscan/package.html
@@ -140,8 +140,8 @@
 
 <script type="text/javascript">
   $(".refresh-button").click(function() {
-    var url = "{% url "apply_task" "djeuscan.tasks.scan_upstream_task" %}";
-    $.post(url, {query: $(this).data("package")}, function() {
+    var url = "{% url "refresh_package" "x/x" %}";
+    $.post(url.replace("x/x", $(this).data("package")), function() {
       alert("Submitted!");
     });
   });
diff --git a/euscanwww/djeuscan/urls.py b/euscanwww/djeuscan/urls.py
index 721fad5..31732b4 100644
--- a/euscanwww/djeuscan/urls.py
+++ b/euscanwww/djeuscan/urls.py
@@ -50,7 +50,10 @@ overlays_patterns = patterns('djeuscan.views',
     url(r'^$', 'overlays', name="overlays"),
 )
 
-tasks_patterns = patterns('djcelery.views',
+tasks_patterns = patterns('djeuscan.views',
+    url(r'^refresh_package/(?P<query>(?:[\w+][\w+.-]*/[\w+][\w+.-]*))/$',
+        "refresh_package",
+        name="refresh_package"),
     url(r'^registered_tasks/$', admin_required(registered_tasks),
         name="registered_tasks"),
     url(r'^apply/(?P<task_name>.*)/$', admin_required(apply),
diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py
index d84c39d..2492e72 100644
--- a/euscanwww/djeuscan/views.py
+++ b/euscanwww/djeuscan/views.py
@@ -5,12 +5,14 @@ from annoying.decorators import render_to, ajax_request
 
 from django.http import Http404
 from django.shortcuts import get_object_or_404
+from django.contrib.auth.decorators import login_required
+from django.views.decorators.http import require_POST
 
 from djeuscan.helpers import version_key, packages_from_names
 from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
-    VersionLog
+    VersionLog, RefreshPackageQuery
 from djeuscan.forms import WorldForm, PackagesForm
-from djeuscan.tasks import launchable_tasks
+from djeuscan.tasks import admin_tasks
 from djeuscan import charts
 
 
@@ -262,7 +264,7 @@ def chart_category(request, **kwargs):
 @ajax_request
 def registered_tasks(request):
     data = {}
-    for task in launchable_tasks:
+    for task in admin_tasks:
         argspec = inspect.getargspec(task.run)
         data[task.name] = {
             "args": argspec.args,
@@ -274,3 +276,14 @@ def registered_tasks(request):
                 "defaults_types": [type(x).__name__ for x in argspec.defaults]
             })
     return {"tasks": data}
+
+
+@login_required
+@require_POST
+@ajax_request
+def refresh_package(request, query):
+    obj, created = RefreshPackageQuery.objects.get_or_create(query=query)
+    if not created:
+        obj.priority += 1
+        obj.save()
+    return {"result": "success"}

From a6a87260b1f02d62c1f97058ab1d138ca1eed0ff Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Fri, 8 Jun 2012 14:23:37 +0200
Subject: [PATCH 14/37] euscanwww: Fixed a minor bug in the task launching
 interface

Signed-off-by: volpino <fox91@anche.no>
---
 .../templates/admin/djcelery/taskstate/change_list.html   | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html b/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html
index a4780cd..eff9ec5 100644
--- a/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html
+++ b/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html
@@ -39,8 +39,10 @@
                   var default_arg_i = task.args.length - 1 - i;
                   if (task.defaults[default_arg_i] !== undefined) {
                     default_arg = task.defaults[default_arg_i];
+                    arg_input.val(default_arg);
+
                     if (task.defaults_types && task.defaults_types[default_arg_i] === "bool") {
-                      arg_input.attr("type", "checkbox");
+                      arg_input.attr("type", "checkbox").attr("checked", default_arg).val("true");
                     }
                     else {
                       return true;
@@ -48,8 +50,6 @@
                   }
                 }
 
-                arg_input.val(default_arg);
-
                 $("#task_data").append(
                   $("<label/>").text(arg + ": ").attr("for", arg)
                 ).append(
@@ -68,7 +68,7 @@
             var url = "{% url "apply_task" "task_name" %}";
 
             $.post(url.replace("task_name", task_name), data, function() {
-              location.reload();
+              //location.reload();
             });
         });
 

From 43e996d83d5aa5a3691fe24c6344e8f7ba5ae48a Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Fri, 8 Jun 2012 14:24:17 +0200
Subject: [PATCH 15/37] euscanwww: Fixed the upstream task that was completely
 broken

Signed-off-by: volpino <fox91@anche.no>
---
 .../management/commands/scan_upstream.py       | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/euscanwww/djeuscan/management/commands/scan_upstream.py b/euscanwww/djeuscan/management/commands/scan_upstream.py
index 131ff1a..5cbdcb9 100644
--- a/euscanwww/djeuscan/management/commands/scan_upstream.py
+++ b/euscanwww/djeuscan/management/commands/scan_upstream.py
@@ -21,19 +21,27 @@ class ScanUpstream(object):
     def scan(self, package):
         CONFIG["format"] = "dict"
         output.set_query(package)
-        scan_upstream(package)
+
+        ret = scan_upstream(package)
+        if ret is not None:
+            if len(ret) > 0:
+                for cp, url, version, handler, confidence in ret:
+                    output.result(cp, version, url, handler, confidence)
+
         out = output.get_formatted_output()
         out_json = output.get_formatted_output("json")
 
         try:
-            cp = out["metadata"]["cp"]
+            cpv = out[package]["metadata"]["cpv"]
         except KeyError:
             return {}
 
-        for out in out["result"]:
-            self.store_version(cp, out["version"], " ".join(out["urls"]))
+        obj = self.store_package(cpv)
 
-        self.store_result(cp, out_json)
+        for res in out[package]["result"]:
+            self.store_version(obj, res["version"], " ".join(res["urls"]))
+
+        self.store_result(obj, out_json)
 
         return out
 

From b43bf8fa2857901efb3394109ebc545edafb38c1 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Fri, 8 Jun 2012 14:43:18 +0200
Subject: [PATCH 16/37] euscanwww: Removing duplicated code

Signed-off-by: volpino <fox91@anche.no>
---
 bin/euscan                                    | 21 +++++--------------
 .../management/commands/scan_upstream.py      |  6 +-----
 .../admin/djcelery/taskstate/change_list.html |  5 ++++-
 pym/euscan/scan.py                            | 10 +++++++++
 4 files changed, 20 insertions(+), 22 deletions(-)

diff --git a/bin/euscan b/bin/euscan
index 907902a..7bea5bd 100755
--- a/bin/euscan
+++ b/bin/euscan
@@ -271,26 +271,15 @@ def main():
             output.eerror('%s: %s' % (query, str(err)))
             exit_helper(1)
 
-        if not CONFIG['quiet'] and not CONFIG['format']:
-            print()
-
         if (CONFIG['format'] or CONFIG['quiet']) and isatty:
             on_progress_gen.next()
             print("\n", file=sys.stderr)
 
-        if ret is not None:
-            if len(ret) > 0:
-                for cp, url, version, handler, confidence in ret:
-                    output.result(cp, version, url, handler, confidence)
-
-                if (CONFIG['format'] or CONFIG['quiet']) and isatty:
-                    print("\n", file=sys.stderr)
-
-            elif not CONFIG['quiet']:
-                output.ewarn(
-                    "Didn't find any new version, check package's homepage " +
-                    "for more informations"
-                )
+        if not ret and not CONFIG['quiet']:
+            output.ewarn(
+                "Didn't find any new version, check package's homepage " +
+                "for more informations"
+            )
 
     output.set_query(None)
 
diff --git a/euscanwww/djeuscan/management/commands/scan_upstream.py b/euscanwww/djeuscan/management/commands/scan_upstream.py
index 5cbdcb9..13b5a9d 100644
--- a/euscanwww/djeuscan/management/commands/scan_upstream.py
+++ b/euscanwww/djeuscan/management/commands/scan_upstream.py
@@ -22,11 +22,7 @@ class ScanUpstream(object):
         CONFIG["format"] = "dict"
         output.set_query(package)
 
-        ret = scan_upstream(package)
-        if ret is not None:
-            if len(ret) > 0:
-                for cp, url, version, handler, confidence in ret:
-                    output.result(cp, version, url, handler, confidence)
+        scan_upstream(package)
 
         out = output.get_formatted_output()
         out_json = output.get_formatted_output("json")
diff --git a/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html b/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html
index eff9ec5..0076a6c 100644
--- a/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html
+++ b/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html
@@ -68,10 +68,13 @@
             var url = "{% url "apply_task" "task_name" %}";
 
             $.post(url.replace("task_name", task_name), data, function() {
-              //location.reload();
+              alert("Submitted!");
+              location.reload();
             });
         });
 
+        setTimeout(location.reload, 10000);
+
       });
     })(django.jQuery);
   </script>
diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py
index 0b8f4ad..5d1bf79 100644
--- a/pym/euscan/scan.py
+++ b/pym/euscan/scan.py
@@ -1,4 +1,7 @@
+from __future__ import print_function
+
 import os
+import sys
 from datetime import datetime
 
 import portage
@@ -212,4 +215,11 @@ def scan_upstream(query, on_progress=None):
     if on_progress:
         on_progress(maxval, curval)
 
+    if len(result) > 0:
+        if not (CONFIG['format'] or CONFIG['quiet']):
+            print("\n", file=sys.stderr)
+
+        for cp, url, version, handler, confidence in result:
+            output.result(cp, version, url, handler, confidence)
+
     return result

From 4246c544ada320d0f58ef002a0e25b7e9223f1d2 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Sat, 9 Jun 2012 16:12:17 +0200
Subject: [PATCH 17/37] euscanwww: bug found, celery timeouts on join.
 Commented out

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/tasks.py     | 18 +++++++++++++++---
 euscanwww/euscanwww/settings.py |  2 ++
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index 77064b0..425467b 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -18,6 +18,13 @@ from djeuscan.management.commands.scan_upstream import ScanUpstream, \
     purge_versions as scan_upstream_purge
 
 
+class TaskFailedException(Exception):
+    """
+    Exception for failed tasks
+    """
+    pass
+
+
 def _launch_command(cmd):
     fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
@@ -38,7 +45,8 @@ def _run_in_chunks(task, iterable, n=32):
             for args in chunk
         ])
         result = job.apply_async()
-        output.extend(list(result.join()))
+        # TODO: understand why this causes timeout
+        #output.extend(list(result.join(timeout=3600)))
     return output
 
 
@@ -114,7 +122,10 @@ def scan_upstream_task(query):
     logger.info("Starting upstream scanning for package %s ...", query)
 
     scan_upstream = ScanUpstream()
-    return scan_upstream.scan(query)
+    result = scan_upstream.scan(query)
+    if not result:
+        raise TaskFailedException("Couldn't scan upstream for this package")
+    return result
 
 
 @task
@@ -127,7 +138,8 @@ def scan_upstream_all_task(purge=False):
     output = _run_in_chunks(
         scan_upstream_task,
         [('%s/%s' % (pkg.category, pkg.name), )
-         for pkg in Package.objects.all()]
+         for pkg in Package.objects.all()],
+        n=16
     )
 
     if purge:
diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py
index 46182c0..d5f0ff6 100644
--- a/euscanwww/euscanwww/settings.py
+++ b/euscanwww/euscanwww/settings.py
@@ -213,6 +213,8 @@ import djcelery
 djcelery.setup_loader()
 BROKER_URL = "amqp://guest:guest@localhost:5672//"
 CELERY_RESULT_BACKEND = "amqp"
+BROKER_CONNECTION_TIMEOUT = 600
+CELERYD_CONCURRENCY = 4
 
 
 try:

From d34a0c4992b72f7fcd8c39c7d6b9c592fdf33c5e Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Sat, 9 Jun 2012 17:11:28 +0200
Subject: [PATCH 18/37] euscanwww: included django-registration and tweaked
 templates

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/templates/_base.html       |  9 ++++++
 .../registration/_registration_base.html      |  1 +
 .../templates/registration/activate.html      | 13 +++++++++
 .../registration/activation_complete.html     |  6 ++++
 .../registration/activation_email.txt         | 11 ++++++++
 .../registration/activation_email_subject.txt |  1 +
 .../templates/registration/login.html         | 28 +++++++++++++++++++
 .../templates/registration/logout.html        | 13 +++++++++
 .../registration/password_change_done.html    |  6 ++++
 .../registration/password_change_form.html    | 16 +++++++++++
 .../registration/password_reset_complete.html |  7 +++++
 .../registration/password_reset_confirm.html  | 15 ++++++++++
 .../registration/password_reset_done.html     | 10 +++++++
 .../registration/password_reset_email.html    | 16 +++++++++++
 .../registration/password_reset_form.html     | 18 ++++++++++++
 .../registration/registration_complete.html   |  6 ++++
 .../registration/registration_form.html       | 19 +++++++++++++
 euscanwww/euscanwww/settings.py               |  2 ++
 euscanwww/euscanwww/urls.py                   |  1 +
 setup.py                                      |  3 +-
 20 files changed, 200 insertions(+), 1 deletion(-)
 create mode 100644 euscanwww/djeuscan/templates/registration/_registration_base.html
 create mode 100644 euscanwww/djeuscan/templates/registration/activate.html
 create mode 100644 euscanwww/djeuscan/templates/registration/activation_complete.html
 create mode 100644 euscanwww/djeuscan/templates/registration/activation_email.txt
 create mode 100644 euscanwww/djeuscan/templates/registration/activation_email_subject.txt
 create mode 100644 euscanwww/djeuscan/templates/registration/login.html
 create mode 100644 euscanwww/djeuscan/templates/registration/logout.html
 create mode 100644 euscanwww/djeuscan/templates/registration/password_change_done.html
 create mode 100644 euscanwww/djeuscan/templates/registration/password_change_form.html
 create mode 100644 euscanwww/djeuscan/templates/registration/password_reset_complete.html
 create mode 100644 euscanwww/djeuscan/templates/registration/password_reset_confirm.html
 create mode 100644 euscanwww/djeuscan/templates/registration/password_reset_done.html
 create mode 100644 euscanwww/djeuscan/templates/registration/password_reset_email.html
 create mode 100644 euscanwww/djeuscan/templates/registration/password_reset_form.html
 create mode 100644 euscanwww/djeuscan/templates/registration/registration_complete.html
 create mode 100644 euscanwww/djeuscan/templates/registration/registration_form.html

diff --git a/euscanwww/djeuscan/templates/_base.html b/euscanwww/djeuscan/templates/_base.html
index e2bcd44..052abd0 100644
--- a/euscanwww/djeuscan/templates/_base.html
+++ b/euscanwww/djeuscan/templates/_base.html
@@ -52,6 +52,15 @@
               <li>---</li>
               <li><a href="{% url "api" %}">API</a></li>
               <li><a href="{% url "about" %}">About</a></li>
+              <li>---</li>
+
+              {% if user.is_authenticated %}
+                <li><a href="#"> Welcome {{ user }}</a></li>
+                <li><a href="{% url "django.contrib.auth.views.logout" %}">Logout</a></li>
+              {% else %}
+                <li><a href="{% url "django.contrib.auth.views.login" %}">Login</a></li>
+                <li><a href="{% url "registration.views.register" %}">Register</a></li>
+              {% endif %}
             {% endblock %}
           </ul>
         </div>
diff --git a/euscanwww/djeuscan/templates/registration/_registration_base.html b/euscanwww/djeuscan/templates/registration/_registration_base.html
new file mode 100644
index 0000000..9789436
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/_registration_base.html
@@ -0,0 +1 @@
+{% extends "_base.html" %}
diff --git a/euscanwww/djeuscan/templates/registration/activate.html b/euscanwww/djeuscan/templates/registration/activate.html
new file mode 100644
index 0000000..2e47513
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/activate.html
@@ -0,0 +1,13 @@
+{% extends "registration/_registration_base.html" %}
+
+{% block title %}{% if account %}Activation complete{% else %}Activation problem{% endif %}{% endblock %}
+
+{% block content %}
+  {% if account %}
+    Thanks {{ account }}, activation complete! You may now <a href='{% url auth_login %}'>login</a>
+    using the username and password you set at registration.
+  {% else %}
+    Oops &ndash; it seems that your activation key is invalid.
+    Please check the url again.
+  {% endif %}
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/registration/activation_complete.html b/euscanwww/djeuscan/templates/registration/activation_complete.html
new file mode 100644
index 0000000..9d53ca9
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/activation_complete.html
@@ -0,0 +1,6 @@
+{% extends "registration/_registration_base.html" %}
+{% block title %}Activation complete{% endblock %}
+{% block content %}
+  Thanks, activation complete! You may now <a href='{% url auth_login %}'>login</a>
+  using the username and password you set at registration.
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/registration/activation_email.txt b/euscanwww/djeuscan/templates/registration/activation_email.txt
new file mode 100644
index 0000000..2e4d942
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/activation_email.txt
@@ -0,0 +1,11 @@
+You (or someone pretending to be you) have asked to register an account at
+{{ site.name }}. If this wasn't you, please ignore this email
+and your address will be removed from our records.
+
+To activate this account, please click the following link within the next
+{{ expiration_days }} days:
+
+http://{{site.domain}}{% url registration_activate activation_key %}
+
+Sincerely,
+{{ site.name }} Management
diff --git a/euscanwww/djeuscan/templates/registration/activation_email_subject.txt b/euscanwww/djeuscan/templates/registration/activation_email_subject.txt
new file mode 100644
index 0000000..13912aa
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/activation_email_subject.txt
@@ -0,0 +1 @@
+Account registration for {{ site.name }}
diff --git a/euscanwww/djeuscan/templates/registration/login.html b/euscanwww/djeuscan/templates/registration/login.html
new file mode 100644
index 0000000..d2e3547
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/login.html
@@ -0,0 +1,28 @@
+{% extends "registration/_registration_base.html" %}
+{% load url from future %}
+{% load i18n %}
+
+{% block content %}
+
+  {% if form.errors %}
+    <p>Your username and password didn't match. Please try again.</p>
+  {% endif %}
+
+  <form method="post" action="{% url 'django.contrib.auth.views.login' %}">
+    {% csrf_token %}
+    <table>
+      <tr>
+        <td>{{ form.username.label_tag }}</td>
+        <td>{{ form.username }}</td>
+      </tr>
+      <tr>
+        <td>{{ form.password.label_tag }}</td>
+        <td>{{ form.password }}</td>
+      </tr>
+    </table>
+
+    <input type="submit" value="login" />
+    <input type="hidden" name="next" value="/" />
+  </form>
+
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/registration/logout.html b/euscanwww/djeuscan/templates/registration/logout.html
new file mode 100644
index 0000000..03746d0
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/logout.html
@@ -0,0 +1,13 @@
+{% extends "registration/_registration_base.html" %}
+{% load i18n %}
+
+{% block content %}
+
+  <p>You're logged out."</p>
+  <p>Redirect...</p>
+
+  <script type="text/javascript">
+    window.location.href = "/";
+  </script>
+
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/registration/password_change_done.html b/euscanwww/djeuscan/templates/registration/password_change_done.html
new file mode 100644
index 0000000..e223576
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/password_change_done.html
@@ -0,0 +1,6 @@
+{% extends "registration/_registration_base.html" %}
+{% block title %}Password changed{% endblock %}
+
+{% block content %}
+  Password successfully changed!
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/registration/password_change_form.html b/euscanwww/djeuscan/templates/registration/password_change_form.html
new file mode 100644
index 0000000..3bb9bb8
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/password_change_form.html
@@ -0,0 +1,16 @@
+{% extends "registration/_registration_base.html" %}
+{% block title %}Change password{% endblock %}
+
+{% block content %}
+  <form method='post' action=''>{% csrf_token %}
+    <table>
+      {{ form }}
+      <tr>
+        <td></td>
+        <td>
+          <input type='submit' value="Change password" />
+        </td>
+      </tr>
+    </table>
+  </form>
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/registration/password_reset_complete.html b/euscanwww/djeuscan/templates/registration/password_reset_complete.html
new file mode 100644
index 0000000..932dabf
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/password_reset_complete.html
@@ -0,0 +1,7 @@
+{% extends "registration/_registration_base.html" %}
+
+{% block title %}Password reset complete{% endblock %}
+
+{% block content %}
+  Your password has been reset! You may now <a href="{{ login_url }}">log in</a>.
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/registration/password_reset_confirm.html b/euscanwww/djeuscan/templates/registration/password_reset_confirm.html
new file mode 100644
index 0000000..653319b
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/password_reset_confirm.html
@@ -0,0 +1,15 @@
+{% extends "registration/_registration_base.html" %}
+{% block title %}Confirm password reset{% endblock %}
+
+{% block content %}
+  Enter your new password below to reset your password:
+  <form method="post" action="">{% csrf_token %}
+    <table>
+      {{ form.as_table }}
+      <tr>
+        <td></td>
+        <td><input type="submit" value="Set password" /></td>
+      </tr>
+    </table>
+  </form>
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/registration/password_reset_done.html b/euscanwww/djeuscan/templates/registration/password_reset_done.html
new file mode 100644
index 0000000..a263a5f
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/password_reset_done.html
@@ -0,0 +1,10 @@
+{% extends "registration/_registration_base.html" %}
+
+{% block title %}Password reset{% endblock %}
+
+{% block content %}
+  <p>
+    We have sent you an email with a link to reset your password. Please check
+    your email and click the link to continue.
+  </p>
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/registration/password_reset_email.html b/euscanwww/djeuscan/templates/registration/password_reset_email.html
new file mode 100644
index 0000000..54fefff
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/password_reset_email.html
@@ -0,0 +1,16 @@
+Greetings {% if user.get_full_name %}{{ user.get_full_name }}{% else %}{{ user }}{% endif %},
+
+You are receiving this email because you (or someone pretending to be you)
+requested that your password be reset on the {{ domain }} site. If you do not
+wish to reset your password, please ignore this message.
+
+To reset your password, please click the following link, or copy and paste it
+into your web browser:
+
+{{ protocol }}://{{ domain }}{% url auth_password_reset_confirm uid token %}
+
+Your username, in case you've forgotten: {{ user.username }}
+
+
+Best regards,
+{{ site_name }} Management
diff --git a/euscanwww/djeuscan/templates/registration/password_reset_form.html b/euscanwww/djeuscan/templates/registration/password_reset_form.html
new file mode 100644
index 0000000..ae5c25b
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/password_reset_form.html
@@ -0,0 +1,18 @@
+{% extends "registration/_registration_base.html" %}
+
+{% block title %}Reset password{% endblock %}
+
+{% block content %}
+  Forgot your password? Enter your email in the form below and we'll send you
+  instructions for creating a new one.
+
+  <form method='post' action=''>{% csrf_token %}
+    <table>
+      {{ form }}
+      <tr>
+        <td></td>
+        <td><input type='submit' value="Reset password" /></td>
+      </tr>
+    </table>
+  </form>
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/registration/registration_complete.html b/euscanwww/djeuscan/templates/registration/registration_complete.html
new file mode 100644
index 0000000..ae8dfcb
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/registration_complete.html
@@ -0,0 +1,6 @@
+{% extends "registration/_registration_base.html" %}
+{% block title %}Activation email sent{% endblock %}
+
+{% block content %}
+  An activation email has been sent. Please check your email and click on the link to activate your account.
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/registration/registration_form.html b/euscanwww/djeuscan/templates/registration/registration_form.html
new file mode 100644
index 0000000..c9ae8e4
--- /dev/null
+++ b/euscanwww/djeuscan/templates/registration/registration_form.html
@@ -0,0 +1,19 @@
+{% extends "registration/_registration_base.html" %}
+
+{% block title %}
+  Register for an account
+{% endblock %}
+
+{% block content %}
+  <table>
+    <form method='post' action=''>{% csrf_token %}
+      {{ form }}
+      <tr>
+        <td></td>
+        <td>
+          <input type="submit" value="Send activation email" />
+        </td>
+      </tr>
+    </form>
+  </table>
+{% endblock %}
diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py
index d5f0ff6..6cae613 100644
--- a/euscanwww/euscanwww/settings.py
+++ b/euscanwww/euscanwww/settings.py
@@ -170,6 +170,7 @@ INSTALLED_APPS = (
     # 'django.contrib.admindocs',
     'south',
     'djcelery',
+    'registration',
 )
 
 # A sample logging configuration. The only tangible logging
@@ -201,6 +202,7 @@ LOGGING = {
     }
 }
 
+ACCOUNT_ACTIVATION_DAYS = 7
 
 PORTAGE_ROOT = "/usr/portage/"
 PORTAGE_CONFIGROOT = PORTAGE_ROOT
diff --git a/euscanwww/euscanwww/urls.py b/euscanwww/euscanwww/urls.py
index 767d375..dffb5a3 100644
--- a/euscanwww/euscanwww/urls.py
+++ b/euscanwww/euscanwww/urls.py
@@ -10,6 +10,7 @@ urlpatterns = patterns('',
     # (r'^admin/doc/', include('django.contrib.admindocs.urls')),
 
     url(r'^admin/', include(admin.site.urls)),
+    url(r'^accounts/', include('registration.backends.default.urls')),
     url(r'^', include('djeuscan.urls')),
 )
 
diff --git a/setup.py b/setup.py
index 240f459..c6adb5f 100755
--- a/setup.py
+++ b/setup.py
@@ -89,7 +89,8 @@ setup(
     ),
     install_requires=['Django==1.4', 'django-annoying==0..7.6', 'South==0.7.4',
                       'django-piston==0.2.3', 'BeautifulSoup==3.2.1',
-                      'matplotlib==1.1.0', 'django-celery==2.5.5'],
+                      'matplotlib==1.1.0', 'django-celery==2.5.5',
+                      'django-registration==0.8'],
     package_dir={'': 'pym'},
     packages=packages,
     package_data={},

From bd9af90ff60908b5495272410cbc94aef06af82a Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Tue, 12 Jun 2012 13:05:58 +0200
Subject: [PATCH 19/37] euscanwww: Removed useless purge tasks, some error
 handling

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/tasks.py     | 35 +++++++++++++++++----------------
 euscanwww/euscanwww/settings.py |  2 +-
 2 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index 425467b..a9341d4 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -26,6 +26,9 @@ class TaskFailedException(Exception):
 
 
 def _launch_command(cmd):
+    """
+    Helper for launching shell commands inside tasks
+    """
     fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
     output = StringIO(fp.communicate()[0])
@@ -33,11 +36,18 @@ def _launch_command(cmd):
 
 
 def _chunks(it, n):
+    """
+    Chunk generator, takes an iterator and the desired size of the chunk
+    """
     for first in it:
         yield [first] + list(islice(it, n - 1))
 
 
 def _run_in_chunks(task, iterable, n=32):
+    """
+    Runs the given task with the given iterable of args in chunks of
+    n subtasks
+    """
     output = []
     for chunk in _chunks(iter(iterable), n):
         job = TaskSet(tasks=[
@@ -46,7 +56,7 @@ def _run_in_chunks(task, iterable, n=32):
         ])
         result = job.apply_async()
         # TODO: understand why this causes timeout
-        #output.extend(list(result.join(timeout=3600)))
+        output.extend(list(result.join(timeout=3600)))
     return output
 
 
@@ -66,7 +76,10 @@ def scan_metadata_task(query, obj=None):
     logger.info("Starting metadata scanning for package %s ...", query)
 
     scan_metadata = ScanMetadata()
-    return scan_metadata.scan(query, obj)
+    result = scan_metadata.scan(query, obj)
+    if not result:
+        raise TaskFailedException("Couldn't scan metadata")
+    return result
 
 
 @task
@@ -111,11 +124,6 @@ def scan_portage_all_task(purge=False):
         scan_portage_purge()
 
 
-@task
-def scan_portage_purge_task():
-    scan_portage_purge()
-
-
 @task
 def scan_upstream_task(query):
     logger = scan_upstream_task.get_logger()
@@ -123,8 +131,8 @@ def scan_upstream_task(query):
 
     scan_upstream = ScanUpstream()
     result = scan_upstream.scan(query)
-    if not result:
-        raise TaskFailedException("Couldn't scan upstream for this package")
+    if not result or result == {}:
+        raise TaskFailedException("Couldn't scan upstream")
     return result
 
 
@@ -143,16 +151,11 @@ def scan_upstream_all_task(purge=False):
     )
 
     if purge:
-        output += [scan_upstream_purge_task()]
+        output += [scan_upstream_purge()]
 
     return output
 
 
-@task
-def scan_upstream_purge_task():
-    scan_upstream_purge()
-
-
 @task
 def emerge_sync():
     cmd = ["emerge", "--sync", "--root", settings.PORTAGE_ROOT,
@@ -201,10 +204,8 @@ admin_tasks = [
     scan_metadata_all_task,
     scan_portage_all_task,
     scan_portage_list_task,
-    scan_portage_purge_task,
     scan_upstream_all_task,
     scan_upstream_list_task,
-    scan_upstream_purge_task,
     emerge_sync,
     layman_sync,
     emerge_regen,
diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py
index 6cae613..733c72c 100644
--- a/euscanwww/euscanwww/settings.py
+++ b/euscanwww/euscanwww/settings.py
@@ -215,7 +215,7 @@ import djcelery
 djcelery.setup_loader()
 BROKER_URL = "amqp://guest:guest@localhost:5672//"
 CELERY_RESULT_BACKEND = "amqp"
-BROKER_CONNECTION_TIMEOUT = 600
+BROKER_CONNECTION_TIMEOUT = 3600
 CELERYD_CONCURRENCY = 4
 
 

From 7367596e33e622a50b76e9ad4c7b87fda8f7457f Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Tue, 12 Jun 2012 13:39:05 +0200
Subject: [PATCH 20/37] euscanwww: Added LDAP login

Needs testing and the settings must be changed to use the gentoo ldap
server

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/euscanwww/settings.py | 12 +++++++++++-
 setup.py                        | 10 ++++++----
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py
index 733c72c..2e7628c 100644
--- a/euscanwww/euscanwww/settings.py
+++ b/euscanwww/euscanwww/settings.py
@@ -202,14 +202,16 @@ LOGGING = {
     }
 }
 
+# django-registration
 ACCOUNT_ACTIVATION_DAYS = 7
+EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 
+# djeuscan tasks
 PORTAGE_ROOT = "/usr/portage/"
 PORTAGE_CONFIGROOT = PORTAGE_ROOT
 LAYMAN_CONFIG = "/etc/layman/layman.cfg"
 EMERGE_REGEN_JOBS = 4
 
-
 # Celery config
 import djcelery
 djcelery.setup_loader()
@@ -218,6 +220,14 @@ CELERY_RESULT_BACKEND = "amqp"
 BROKER_CONNECTION_TIMEOUT = 3600
 CELERYD_CONCURRENCY = 4
 
+# LDAP authentication
+# TODO: Test data - change me!
+AUTH_LDAP_SERVER_URI = "ldap://localhost"
+AUTH_LDAP_USER_DN_TEMPLATE = "uid=%(user)s,ou=users,dc=my-domain,dc=com"
+AUTHENTICATION_BACKENDS = (
+    'django_auth_ldap.backend.LDAPBackend',
+    'django.contrib.auth.backends.ModelBackend',
+)
 
 try:
     from local_settings import *
diff --git a/setup.py b/setup.py
index c6adb5f..2954d54 100755
--- a/setup.py
+++ b/setup.py
@@ -87,10 +87,12 @@ setup(
         'https://github.com/iksaif/euscan/tarball/' +
         ('master' if __version__ == '9999' else ('euscan-%s' % __version__))
     ),
-    install_requires=['Django==1.4', 'django-annoying==0..7.6', 'South==0.7.4',
-                      'django-piston==0.2.3', 'BeautifulSoup==3.2.1',
-                      'matplotlib==1.1.0', 'django-celery==2.5.5',
-                      'django-registration==0.8'],
+    install_requires=[
+        'Django==1.4', 'django-annoying==0.7.6', 'South==0.7.4',
+        'django-piston==0.2.3', 'BeautifulSoup==3.2.1', 'matplotlib==1.1.0',
+        'django-celery==2.5.5', 'django-registration==0.8',
+        'python-ldap==2.4.10', 'django-auth-ldap==1.1',
+    ],
     package_dir={'': 'pym'},
     packages=packages,
     package_data={},

From bd971a7da0161d9678aaac12e1af73fe2d81ab3d Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Tue, 12 Jun 2012 14:55:58 +0200
Subject: [PATCH 21/37] euscanwww: When scanning upstream output object was
 never cleaned

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/tasks.py | 4 ++++
 pym/euscan/out.py           | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index a9341d4..897f9b6 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -8,6 +8,8 @@ from celery.task.sets import TaskSet
 
 from django.conf import settings
 
+from euscan import output as euscan_output
+
 from djeuscan.models import Package, RefreshPackageQuery
 from djeuscan.management.commands.regen_rrds import regen_rrds
 from djeuscan.management.commands.update_counters import update_counters
@@ -129,8 +131,10 @@ def scan_upstream_task(query):
     logger = scan_upstream_task.get_logger()
     logger.info("Starting upstream scanning for package %s ...", query)
 
+    euscan_output.clean()
     scan_upstream = ScanUpstream()
     result = scan_upstream.scan(query)
+    euscan_output.clean()
     if not result or result == {}:
         raise TaskFailedException("Couldn't scan upstream")
     return result
diff --git a/pym/euscan/out.py b/pym/euscan/out.py
index 6a98336..1a61579 100644
--- a/pym/euscan/out.py
+++ b/pym/euscan/out.py
@@ -76,6 +76,10 @@ class EuscanOutput(object):
         self.queries = defaultdict(dict)
         self.current_query = None
 
+    def clean(self):
+        self.queries = defaultdict(dict)
+        self.current_query = None
+
     def set_query(self, query):
         self.current_query = query
         if query is None:

From 11c234f7f79251a9862cef077d3fbfe6603f4414 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Tue, 12 Jun 2012 15:27:28 +0200
Subject: [PATCH 22/37] euscanwww: Added account profile page

* Added an account profile page
* Added a button to favourite packages
* Implemented models to favourite categories, herds, maintainers but
  still to implement

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/admin.py                   |  10 +-
 euscanwww/djeuscan/managers.py                |   1 -
 ...add_unique_categoryassociation_user_cat.py | 238 ++++++++++++++++++
 euscanwww/djeuscan/models.py                  |  46 ++++
 euscanwww/djeuscan/templates/_base.html       |  30 ++-
 .../templates/euscan/accounts/categories.html |  11 +
 .../templates/euscan/accounts/herds.html      |  11 +
 .../templates/euscan/accounts/index.html      |  11 +
 .../euscan/accounts/maintainers.html          |  11 +
 .../templates/euscan/accounts/packages.html   |  16 ++
 .../djeuscan/templates/euscan/package.html    |  11 +-
 euscanwww/djeuscan/urls.py                    |  24 +-
 euscanwww/djeuscan/views.py                   |  83 +++++-
 euscanwww/htdocs/css/style.css                |  11 +-
 14 files changed, 491 insertions(+), 23 deletions(-)
 create mode 100644 euscanwww/djeuscan/migrations/0004_auto__add_categoryassociation__add_unique_categoryassociation_user_cat.py
 create mode 100644 euscanwww/djeuscan/templates/euscan/accounts/categories.html
 create mode 100644 euscanwww/djeuscan/templates/euscan/accounts/herds.html
 create mode 100644 euscanwww/djeuscan/templates/euscan/accounts/index.html
 create mode 100644 euscanwww/djeuscan/templates/euscan/accounts/maintainers.html
 create mode 100644 euscanwww/djeuscan/templates/euscan/accounts/packages.html

diff --git a/euscanwww/djeuscan/admin.py b/euscanwww/djeuscan/admin.py
index 329117a..cec9244 100644
--- a/euscanwww/djeuscan/admin.py
+++ b/euscanwww/djeuscan/admin.py
@@ -1,5 +1,7 @@
 from djeuscan.models import Package, Version, VersionLog, EuscanResult, \
-    Log, WorldLog, CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer
+    Log, WorldLog, CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer, \
+    RefreshPackageQuery, HerdAssociation, CategoryAssociation, \
+    MaintainerAssociation, PackageAssociation
 from django.contrib import admin
 
 
@@ -44,3 +46,9 @@ admin.site.register(WorldLog)
 admin.site.register(CategoryLog)
 admin.site.register(HerdLog)
 admin.site.register(MaintainerLog)
+
+admin.site.register(RefreshPackageQuery)
+admin.site.register(HerdAssociation)
+admin.site.register(CategoryAssociation)
+admin.site.register(MaintainerAssociation)
+admin.site.register(PackageAssociation)
diff --git a/euscanwww/djeuscan/managers.py b/euscanwww/djeuscan/managers.py
index 2172602..0d4c544 100644
--- a/euscanwww/djeuscan/managers.py
+++ b/euscanwww/djeuscan/managers.py
@@ -35,7 +35,6 @@ ANNOTATE_DICT = {name: models.Sum(name)
 
 
 class PackageMixin(object):
-
     for_maintainer = _gen_for_function('maintainers')
     for_herd = _gen_for_function('herds')
     for_category = _gen_for_function('category')
diff --git a/euscanwww/djeuscan/migrations/0004_auto__add_categoryassociation__add_unique_categoryassociation_user_cat.py b/euscanwww/djeuscan/migrations/0004_auto__add_categoryassociation__add_unique_categoryassociation_user_cat.py
new file mode 100644
index 0000000..888fefa
--- /dev/null
+++ b/euscanwww/djeuscan/migrations/0004_auto__add_categoryassociation__add_unique_categoryassociation_user_cat.py
@@ -0,0 +1,238 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'CategoryAssociation'
+        db.create_table('djeuscan_categoryassociation', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('category', self.gf('django.db.models.fields.CharField')(max_length=128)),
+        ))
+        db.send_create_signal('djeuscan', ['CategoryAssociation'])
+
+        # Adding unique constraint on 'CategoryAssociation', fields ['user', 'category']
+        db.create_unique('djeuscan_categoryassociation', ['user_id', 'category'])
+
+        # Adding model 'PackageAssociation'
+        db.create_table('djeuscan_packageassociation', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('package', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Package'])),
+        ))
+        db.send_create_signal('djeuscan', ['PackageAssociation'])
+
+        # Adding unique constraint on 'PackageAssociation', fields ['user', 'package']
+        db.create_unique('djeuscan_packageassociation', ['user_id', 'package_id'])
+
+        # Adding model 'HerdAssociation'
+        db.create_table('djeuscan_herdassociation', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('herd', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Herd'])),
+        ))
+        db.send_create_signal('djeuscan', ['HerdAssociation'])
+
+        # Adding unique constraint on 'HerdAssociation', fields ['user', 'herd']
+        db.create_unique('djeuscan_herdassociation', ['user_id', 'herd_id'])
+
+        # Adding model 'MaintainerAssociation'
+        db.create_table('djeuscan_maintainerassociation', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('maintainer', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Maintainer'])),
+        ))
+        db.send_create_signal('djeuscan', ['MaintainerAssociation'])
+
+        # Adding unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer']
+        db.create_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id'])
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer']
+        db.delete_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id'])
+
+        # Removing unique constraint on 'HerdAssociation', fields ['user', 'herd']
+        db.delete_unique('djeuscan_herdassociation', ['user_id', 'herd_id'])
+
+        # Removing unique constraint on 'PackageAssociation', fields ['user', 'package']
+        db.delete_unique('djeuscan_packageassociation', ['user_id', 'package_id'])
+
+        # Removing unique constraint on 'CategoryAssociation', fields ['user', 'category']
+        db.delete_unique('djeuscan_categoryassociation', ['user_id', 'category'])
+
+        # Deleting model 'CategoryAssociation'
+        db.delete_table('djeuscan_categoryassociation')
+
+        # Deleting model 'PackageAssociation'
+        db.delete_table('djeuscan_packageassociation')
+
+        # Deleting model 'HerdAssociation'
+        db.delete_table('djeuscan_herdassociation')
+
+        # Deleting model 'MaintainerAssociation'
+        db.delete_table('djeuscan_maintainerassociation')
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'djeuscan.categoryassociation': {
+            'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.categorylog': {
+            'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.euscanresult': {
+            'Meta': {'object_name': 'EuscanResult'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'djeuscan.herd': {
+            'Meta': {'object_name': 'Herd'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
+            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'djeuscan.herdassociation': {
+            'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.herdlog': {
+            'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.log': {
+            'Meta': {'object_name': 'Log'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'djeuscan.maintainer': {
+            'Meta': {'object_name': 'Maintainer'},
+            'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.maintainerassociation': {
+            'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.maintainerlog': {
+            'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
+        },
+        'djeuscan.package': {
+            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
+            'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.packageassociation': {
+            'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.refreshpackagequery': {
+            'Meta': {'object_name': 'RefreshPackageQuery'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
+        },
+        'djeuscan.version': {
+            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
+            'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.versionlog': {
+            'Meta': {'object_name': 'VersionLog'},
+            'action': ('django.db.models.fields.IntegerField', [], {}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.worldlog': {
+            'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        }
+    }
+
+    complete_apps = ['djeuscan']
\ No newline at end of file
diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py
index 85a0c4a..936b2fc 100644
--- a/euscanwww/djeuscan/models.py
+++ b/euscanwww/djeuscan/models.py
@@ -2,6 +2,8 @@ from django.db import models
 from django.core.validators import RegexValidator, validate_email, URLValidator
 from django.core.exceptions import ValidationError
 
+from django.contrib.auth.models import User
+
 from djeuscan.managers import PackageManager, VersionLogManager, \
     EuscanResultManager
 
@@ -268,3 +270,47 @@ class RefreshPackageQuery(models.Model):
 
     def __unicode__(self):
         return u'[%d] %s' % (self.priority, self.query)
+
+
+class HerdAssociation(models.Model):
+    user = models.ForeignKey(User)
+    herd = models.ForeignKey(Herd)
+
+    class Meta:
+        unique_together = ['user', 'herd']
+
+    def __unicode__(self):
+        return u'[%s] %s' % (self.user, self.herd)
+
+
+class MaintainerAssociation(models.Model):
+    user = models.ForeignKey(User)
+    maintainer = models.ForeignKey(Maintainer)
+
+    class Meta:
+        unique_together = ['user', 'maintainer']
+
+    def __unicode__(self):
+        return u'[%s] %s' % (self.user, self.maintainer)
+
+
+class PackageAssociation(models.Model):
+    user = models.ForeignKey(User)
+    package = models.ForeignKey(Package)
+
+    class Meta:
+        unique_together = ['user', 'package']
+
+    def __unicode__(self):
+        return u'[%s] %s' % (self.user, self.package)
+
+
+class CategoryAssociation(models.Model):
+    user = models.ForeignKey(User)
+    category = models.CharField(max_length=128, validators=[validate_category])
+
+    class Meta:
+        unique_together = ['user', 'category']
+
+    def __unicode__(self):
+        return u'[%s] %s' % (self.user, self.category)
diff --git a/euscanwww/djeuscan/templates/_base.html b/euscanwww/djeuscan/templates/_base.html
index 052abd0..afd9324 100644
--- a/euscanwww/djeuscan/templates/_base.html
+++ b/euscanwww/djeuscan/templates/_base.html
@@ -37,12 +37,25 @@
               <li><a href="{% url "overlays" %}">Overlays</a></li>
               <li><a href="{% url "world" %}">Scan World</a></li>
               <li><a href="{% url "statistics" %}">Statistics</a></li>
-              <!--
+
               <li>---</li>
-              <li><a href="#">Login</a></li>
-              <li><a href="#">Register</a></li>
-              -->
+
+              {% if user.is_authenticated %}
+                <li><a href="{% url "accounts_index" %}">{{ user }}'s profile</a></li>
+                <ul class="submenu">
+                  <li><a href="{% url "accounts_categories" %}">Categories</a></li>
+                  <li><a href="{% url "accounts_herds" %}">Herds</a></li>
+                  <li><a href="{% url "accounts_maintainers" %}">Maintainer profiles</a></li>
+                  <li><a href="{% url "accounts_packages" %}">Packages</a></li>
+                </ul>
+                <li><a href="{% url "django.contrib.auth.views.logout" %}">Logout</a></li>
+              {% else %}
+              <li><a href="{% url "django.contrib.auth.views.login" %}?next={% url "accounts_index" %}">Login</a></li>
+                <li><a href="{% url "registration.views.register" %}">Register</a></li>
+              {% endif %}
+
               <li>---</li>
+
               {% block menu_feed %}
                 <li>
                   <img src="{{ STATIC_URL }}img/feed.png" alt="feed" />
@@ -52,15 +65,6 @@
               <li>---</li>
               <li><a href="{% url "api" %}">API</a></li>
               <li><a href="{% url "about" %}">About</a></li>
-              <li>---</li>
-
-              {% if user.is_authenticated %}
-                <li><a href="#"> Welcome {{ user }}</a></li>
-                <li><a href="{% url "django.contrib.auth.views.logout" %}">Logout</a></li>
-              {% else %}
-                <li><a href="{% url "django.contrib.auth.views.login" %}">Login</a></li>
-                <li><a href="{% url "registration.views.register" %}">Register</a></li>
-              {% endif %}
             {% endblock %}
           </ul>
         </div>
diff --git a/euscanwww/djeuscan/templates/euscan/accounts/categories.html b/euscanwww/djeuscan/templates/euscan/accounts/categories.html
new file mode 100644
index 0000000..a2bd6fa
--- /dev/null
+++ b/euscanwww/djeuscan/templates/euscan/accounts/categories.html
@@ -0,0 +1,11 @@
+{% extends "_base.html" %}
+{% load url from future %}
+
+{% block title %}
+  {{ block.super }} - My categories
+{% endblock %}
+
+
+{% block content %}
+  <h2>Welcome {{ user }}</h2>
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/euscan/accounts/herds.html b/euscanwww/djeuscan/templates/euscan/accounts/herds.html
new file mode 100644
index 0000000..d77aa03
--- /dev/null
+++ b/euscanwww/djeuscan/templates/euscan/accounts/herds.html
@@ -0,0 +1,11 @@
+{% extends "_base.html" %}
+{% load url from future %}
+
+{% block title %}
+  {{ block.super }} - My herds
+{% endblock %}
+
+
+{% block content %}
+  <h2>Welcome {{ user }}</h2>
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/euscan/accounts/index.html b/euscanwww/djeuscan/templates/euscan/accounts/index.html
new file mode 100644
index 0000000..46ba5c7
--- /dev/null
+++ b/euscanwww/djeuscan/templates/euscan/accounts/index.html
@@ -0,0 +1,11 @@
+{% extends "_base.html" %}
+{% load url from future %}
+
+{% block title %}
+  {{ block.super }} - Welcome {{ user }}
+{% endblock %}
+
+
+{% block content %}
+  <h2>Welcome {{ user }}</h2>
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/euscan/accounts/maintainers.html b/euscanwww/djeuscan/templates/euscan/accounts/maintainers.html
new file mode 100644
index 0000000..1a907b9
--- /dev/null
+++ b/euscanwww/djeuscan/templates/euscan/accounts/maintainers.html
@@ -0,0 +1,11 @@
+{% extends "_base.html" %}
+{% load url from future %}
+
+{% block title %}
+  {{ block.super }} - My maintainer profiles
+{% endblock %}
+
+
+{% block content %}
+  <h2>Welcome {{ user }}</h2>
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/euscan/accounts/packages.html b/euscanwww/djeuscan/templates/euscan/accounts/packages.html
new file mode 100644
index 0000000..7c847f6
--- /dev/null
+++ b/euscanwww/djeuscan/templates/euscan/accounts/packages.html
@@ -0,0 +1,16 @@
+{% extends "euscan/_datatable.html" %}
+{% load url from future %}
+
+{% load packages %}
+
+{% block title %}
+  {{ block.super }} - My packages
+{% endblock %}
+
+
+{% block content %}
+  <h2>My Packages</h2>
+
+  {% packages packages %}
+
+{% endblock %}
diff --git a/euscanwww/djeuscan/templates/euscan/package.html b/euscanwww/djeuscan/templates/euscan/package.html
index f042d04..6eb500f 100644
--- a/euscanwww/djeuscan/templates/euscan/package.html
+++ b/euscanwww/djeuscan/templates/euscan/package.html
@@ -27,10 +27,13 @@
 <h2>
   {{ package.category }}/{{ package.name }}
 
-  {% if user.is_superuser %}
+  {% if user.is_authenticated %}
     <button class="refresh-button" data-package="{{ package.category }}/{{ package.name }}">
       Refresh
     </button>
+    <button class="favourite-button" data-url="{% url "favourite_package" package.category package.name %}">
+      Add to favourites
+    </button>
   {% endif %}
 </h2>
 <dl>
@@ -145,6 +148,12 @@
       alert("Submitted!");
     });
   });
+
+  $(".favourite-button").click(function() {
+    $.post($(this).data("url"), function() {
+      alert("Favourited!");
+    });
+  });
 </script>
 
 {% endblock %}
diff --git a/euscanwww/djeuscan/urls.py b/euscanwww/djeuscan/urls.py
index 31732b4..17063b6 100644
--- a/euscanwww/djeuscan/urls.py
+++ b/euscanwww/djeuscan/urls.py
@@ -1,7 +1,7 @@
 from django.conf.urls.defaults import url, patterns, include
 from django.contrib.auth.decorators import user_passes_test
 
-from djcelery.views import apply
+from djcelery.views import apply as apply_task
 from djeuscan.views import registered_tasks
 
 from djeuscan.feeds import PackageFeed, CategoryFeed, HerdFeed, \
@@ -16,6 +16,8 @@ package_patterns = patterns('djeuscan.views',
         PackageFeed(), name='package_feed'),
     url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/$',
         'package', name="package"),
+    url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/favourite$',
+        'favourite_package', name="favourite_package"),
 )
 
 categories_patterns = patterns('djeuscan.views',
@@ -25,6 +27,8 @@ categories_patterns = patterns('djeuscan.views',
         name='category_feed'),
     url(r'^(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$',
         'chart_category', name="chart_category"),
+    url(r'^(?P<category>[\w+][\w+.-]*)/favourite$',
+        'favourite_category', name="favourite_category"),
     url(r'^$', 'categories', name="categories"),
 )
 
@@ -33,6 +37,8 @@ herds_patterns = patterns('djeuscan.views',
     url(r'^(?P<herd>[\@\{\}\w+.-]*)/feed/$', HerdFeed(), name='herd_feed'),
     url(r'^(?P<herd>[\@\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$',
         'chart_herd', name="chart_herd"),
+    url(r'^(?P<herd>[\@\{\}\w+.-]*)/favourite$', 'favourite_herd',
+        name="favourite_herd"),
     url(r'^$', 'herds', name="herds"),
 )
 
@@ -42,6 +48,8 @@ maintainers_patterns = patterns('djeuscan.views',
         name='maintainer_feed'),
     url(r'^(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$',
         'chart_maintainer', name="chart_maintainer"),
+    url(r'^(?P<maintainer_id>\d+)/favourite$',
+        'favourite_maintainer', name="favourite_maintainer"),
     url(r'^$', 'maintainers', name="maintainers"),
 )
 
@@ -56,10 +64,20 @@ tasks_patterns = patterns('djeuscan.views',
         name="refresh_package"),
     url(r'^registered_tasks/$', admin_required(registered_tasks),
         name="registered_tasks"),
-    url(r'^apply/(?P<task_name>.*)/$', admin_required(apply),
+    url(r'^apply/(?P<task_name>.*)/$', admin_required(apply_task),
         name="apply_task"),
 )
 
+accounts_patterns = patterns('djeuscan.views',
+    url(r'^profile/$', 'accounts_index', name="accounts_index"),
+    url(r'^categories/$', 'accounts_categories', name="accounts_categories"),
+    url(r'^herds/$', 'accounts_herds', name="accounts_herds"),
+    url(r'^maintainers/$', 'accounts_maintainers',
+        name="accounts_maintainers"),
+    url(r'^packages/$', 'accounts_packages', name="accounts_packages"),
+)
+
+
 urlpatterns = patterns('djeuscan.views',
     # Global stuff
     url(r'^api/', include('djeuscan.api.urls')),
@@ -79,5 +97,7 @@ urlpatterns = patterns('djeuscan.views',
     url(r'^maintainers/', include(maintainers_patterns)),
     url(r'^overlays/', include(overlays_patterns)),
     url(r'^package/', include(package_patterns)),
+
     url(r'^tasks/', include(tasks_patterns)),
+    url(r'^accounts/', include(accounts_patterns)),
 )
diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py
index 2492e72..7a34271 100644
--- a/euscanwww/djeuscan/views.py
+++ b/euscanwww/djeuscan/views.py
@@ -10,7 +10,8 @@ from django.views.decorators.http import require_POST
 
 from djeuscan.helpers import version_key, packages_from_names
 from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
-    VersionLog, RefreshPackageQuery
+    VersionLog, RefreshPackageQuery, HerdAssociation, MaintainerAssociation, \
+    CategoryAssociation, PackageAssociation
 from djeuscan.forms import WorldForm, PackagesForm
 from djeuscan.tasks import admin_tasks
 from djeuscan import charts
@@ -287,3 +288,83 @@ def refresh_package(request, query):
         obj.priority += 1
         obj.save()
     return {"result": "success"}
+
+
+@login_required
+@render_to('euscan/accounts/index.html')
+def accounts_index(request):
+    return {}
+
+
+@login_required
+@render_to('euscan/accounts/categories.html')
+def accounts_categories(request):
+    return {}
+
+
+@login_required
+@render_to('euscan/accounts/herds.html')
+def accounts_herds(request):
+    return {}
+
+
+@login_required
+@render_to('euscan/accounts/maintainers.html')
+def accounts_maintainers(request):
+    return {}
+
+
+@login_required
+@render_to('euscan/accounts/packages.html')
+def accounts_packages(request):
+    packages = [obj.package for obj in
+                PackageAssociation.objects.filter(user=request.user)]
+    return {"packages": packages}
+
+
+@login_required
+@require_POST
+@ajax_request
+def favourite_package(request, category, package):
+    obj = get_object_or_404(Package, category=category, name=package)
+    _, created = PackageAssociation.objects.get_or_create(
+        user=request.user, package=obj
+    )
+    return {"success": created}
+
+
+@login_required
+@require_POST
+@ajax_request
+def favourite_herd(request, herd):
+    obj = get_object_or_404(Herd, herd=herd)
+    _, created = HerdAssociation.objects.get_or_create(
+        user=request.user, herd=obj
+    )
+    return {"success": created}
+
+
+@login_required
+@require_POST
+@ajax_request
+def favourite_maintainer(request, maintainer_id):
+    obj = get_object_or_404(Maintainer, pk=maintainer_id)
+    _, created = MaintainerAssociation.objects.get_or_create(
+        user=request.user, maintainer=obj
+    )
+    return {"success": created}
+
+
+@login_required
+@require_POST
+@ajax_request
+def favourite_category(request, category):
+    packages = Package.objects.for_category(category, last_versions=True)
+
+    if not packages:
+        raise Http404
+
+    _, created = CategoryAssociation.objects.get_or_create(
+        user=request.user, category=category
+    )
+    return {"success": created}
diff --git a/euscanwww/htdocs/css/style.css b/euscanwww/htdocs/css/style.css
index 7ac69a3..62d6cda 100644
--- a/euscanwww/htdocs/css/style.css
+++ b/euscanwww/htdocs/css/style.css
@@ -87,10 +87,13 @@ a:hover, a:active
     color: #fff;
 }
 
-.menu li
-{
-  list-style-type: none;
- }
+.menu li {
+    list-style-type: none;
+}
+
+.submenu {
+    padding-left: 15px;
+}
 
 .menu dl {
     margin-left: 15px;

From 34d3bd714d2bb77ca1c51fe6f52e58c07c671167 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Thu, 14 Jun 2012 10:03:49 +0200
Subject: [PATCH 23/37] euscanwww: Watched herds/maintainers/categories panel

* Refactored templates and templatetags
* Added unfavourite views
* Added button to favourite/unfavourite package
* Panel for showing favourited objects

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/tasks.py                   |   2 +-
 euscanwww/djeuscan/templates/_base.html       |   2 +-
 .../templates/euscan/_categories_table.html   |  32 +++++
 .../templates/euscan/_herds_table.html        |  34 +++++
 .../templates/euscan/_maintainers_table.html  |  40 ++++++
 .../templates/euscan/_package_cols.html       |   2 +-
 .../djeuscan/templates/euscan/_packages.html  |   2 +-
 .../templates/euscan/accounts/categories.html |  10 +-
 .../templates/euscan/accounts/herds.html      |   9 +-
 .../euscan/accounts/maintainers.html          |   9 +-
 .../templates/euscan/accounts/packages.html   |   6 +-
 .../djeuscan/templates/euscan/categories.html |  33 +----
 .../djeuscan/templates/euscan/category.html   |   2 +-
 euscanwww/djeuscan/templates/euscan/herd.html |   2 +-
 .../djeuscan/templates/euscan/herds.html      |  36 +-----
 .../djeuscan/templates/euscan/maintainer.html |   2 +-
 .../templates/euscan/maintainers.html         |  38 +-----
 .../djeuscan/templates/euscan/overlay.html    |   2 +-
 .../djeuscan/templates/euscan/overlays.html   |   2 +-
 .../djeuscan/templates/euscan/package.html    |  15 ++-
 .../djeuscan/templates/euscan/world_scan.html |   2 +-
 euscanwww/djeuscan/templatetags/euscan.py     |  49 +++++++
 euscanwww/djeuscan/templatetags/packages.py   |  21 ---
 euscanwww/djeuscan/urls.py                    |   9 ++
 euscanwww/djeuscan/views.py                   |  71 +++++++++-
 euscanwww/htdocs/css/style.css                | 121 +++++++++---------
 26 files changed, 344 insertions(+), 209 deletions(-)
 create mode 100644 euscanwww/djeuscan/templates/euscan/_categories_table.html
 create mode 100644 euscanwww/djeuscan/templates/euscan/_herds_table.html
 create mode 100644 euscanwww/djeuscan/templates/euscan/_maintainers_table.html
 create mode 100644 euscanwww/djeuscan/templatetags/euscan.py
 delete mode 100644 euscanwww/djeuscan/templatetags/packages.py

diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index 897f9b6..290df65 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -58,7 +58,7 @@ def _run_in_chunks(task, iterable, n=32):
         ])
         result = job.apply_async()
         # TODO: understand why this causes timeout
-        output.extend(list(result.join(timeout=3600)))
+        #output.extend(list(result.join(timeout=3600)))
     return output
 
 
diff --git a/euscanwww/djeuscan/templates/_base.html b/euscanwww/djeuscan/templates/_base.html
index afd9324..54ea007 100644
--- a/euscanwww/djeuscan/templates/_base.html
+++ b/euscanwww/djeuscan/templates/_base.html
@@ -45,7 +45,7 @@
                 <ul class="submenu">
                   <li><a href="{% url "accounts_categories" %}">Categories</a></li>
                   <li><a href="{% url "accounts_herds" %}">Herds</a></li>
-                  <li><a href="{% url "accounts_maintainers" %}">Maintainer profiles</a></li>
+                  <li><a href="{% url "accounts_maintainers" %}">Maintainers</a></li>
                   <li><a href="{% url "accounts_packages" %}">Packages</a></li>
                 </ul>
                 <li><a href="{% url "django.contrib.auth.views.logout" %}">Logout</a></li>
diff --git a/euscanwww/djeuscan/templates/euscan/_categories_table.html b/euscanwww/djeuscan/templates/euscan/_categories_table.html
new file mode 100644
index 0000000..f40e404
--- /dev/null
+++ b/euscanwww/djeuscan/templates/euscan/_categories_table.html
@@ -0,0 +1,32 @@
+{% load url from future %}
+{% load euscan %}
+
+<table id="table" class="display">
+  <thead>
+    <th>Category</th>
+    <th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
+    <th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
+    <th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
+    <th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
+    {% if extras %}
+    <th>Graphs</th>
+    {% endif %}
+  </thead>
+  <tbody>
+  {% for category in categories %}
+  <tr>
+    <td>
+      <a href="{% url "category" category.category %}">{{ category.category }}</a>
+      {% package_bar category %}
+    </td>
+    {% package_cols category %}
+    {% if extras %}
+    <td>
+      <img src="{% url "chart_category" category.category 'packages-monthly-small' %}" />
+      <img src="{% url "chart_category" category.category 'versions-monthly-small' %}" />
+    </td>
+    {% endif %}
+  </tr>
+  {% endfor %}
+  </tbody>
+</table>
diff --git a/euscanwww/djeuscan/templates/euscan/_herds_table.html b/euscanwww/djeuscan/templates/euscan/_herds_table.html
new file mode 100644
index 0000000..ae218a6
--- /dev/null
+++ b/euscanwww/djeuscan/templates/euscan/_herds_table.html
@@ -0,0 +1,34 @@
+{% load url from future %}
+{% load euscan %}
+
+<table id="table" class="display">
+  <thead>
+    <th>Herd</th>
+    <th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
+    <th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
+    <th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
+    <th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
+    {% if extras %}
+    <th>Graphs</th>
+    {% endif %}
+  </thead>
+  <tbody>
+  {% for herd in herds %}
+    <tr>
+      <td>
+        <a href="{% url "herd" herd.herds__herd %}">
+          {{ herd.herds__herd }}
+        </a>
+        {% package_bar herd %}
+      </td>
+      {% package_cols herd %}
+      {% if extras %}
+        <td>
+          <img src="{% url "chart_herd" herd.herds__herd 'packages-monthly-small' %}" />
+          <img src="{% url "chart_herd" herd.herds__herd 'versions-monthly-small' %}" />
+        </td>
+      {% endif %}
+    </tr>
+  {% endfor %}
+  </tbody>
+</table>
diff --git a/euscanwww/djeuscan/templates/euscan/_maintainers_table.html b/euscanwww/djeuscan/templates/euscan/_maintainers_table.html
new file mode 100644
index 0000000..f7b1412
--- /dev/null
+++ b/euscanwww/djeuscan/templates/euscan/_maintainers_table.html
@@ -0,0 +1,40 @@
+{% load url from future %}
+{% load euscan %}
+
+<table id="table" class="display">
+  <thead>
+    <th>Maintainer</th>
+    <th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
+    <th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
+    <th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
+    <th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
+    {% if extras %}
+      <th>Graphs</th>
+    {% endif %}
+  </thead>
+  <tbody>
+  {% for maintainer in maintainers %}
+    <tr>
+      <td>
+        <a href="{% url "maintainer" maintainer.maintainers__id %}">
+          {% if maintainer.maintainers__name != maintainer.maintainers__email %}
+            {{ maintainer.maintainers__name }} &lt;{{ maintainer.maintainers__email }}&gt;
+          {% else %}
+            {{ maintainer.maintainers__name }}
+          {% endif %}
+        </a>
+        {% package_bar maintainer %}
+      </td>
+      {% package_cols maintainer %}
+
+      {% if extras %}
+        <td>
+          <img src="{% url "chart_maintainer" maintainer.maintainers__id 'packages-monthly-small' %}" />
+          <img src="{% url "chart_maintainer" maintainer.maintainers__id 'versions-monthly-small' %}" />
+        </td>
+      {% endif %}
+
+    </tr>
+  {% endfor %}
+  </tbody>
+</table>
diff --git a/euscanwww/djeuscan/templates/euscan/_package_cols.html b/euscanwww/djeuscan/templates/euscan/_package_cols.html
index a3b6942..5140701 100644
--- a/euscanwww/djeuscan/templates/euscan/_package_cols.html
+++ b/euscanwww/djeuscan/templates/euscan/_package_cols.html
@@ -1,4 +1,4 @@
-{% load packages %}
+{% load euscan %}
 {% load mul %}
 {% load sub %}
 {% load div %}
diff --git a/euscanwww/djeuscan/templates/euscan/_packages.html b/euscanwww/djeuscan/templates/euscan/_packages.html
index 58db056..6df68c9 100644
--- a/euscanwww/djeuscan/templates/euscan/_packages.html
+++ b/euscanwww/djeuscan/templates/euscan/_packages.html
@@ -1,4 +1,4 @@
-{% load packages %}
+{% load euscan %}
 {% load sub %}
 {% load div %}
 {% load mul %}
diff --git a/euscanwww/djeuscan/templates/euscan/accounts/categories.html b/euscanwww/djeuscan/templates/euscan/accounts/categories.html
index a2bd6fa..3c823f3 100644
--- a/euscanwww/djeuscan/templates/euscan/accounts/categories.html
+++ b/euscanwww/djeuscan/templates/euscan/accounts/categories.html
@@ -1,11 +1,15 @@
-{% extends "_base.html" %}
+{% extends "euscan/_datatable.html" %}
 {% load url from future %}
+{% load euscan %}
 
 {% block title %}
-  {{ block.super }} - My categories
+  {{ block.super }} - Watched categories
 {% endblock %}
 
 
 {% block content %}
-  <h2>Welcome {{ user }}</h2>
+  <h2>Watched categories</h2>
+
+  {% categories_table categories %}
+
 {% endblock %}
diff --git a/euscanwww/djeuscan/templates/euscan/accounts/herds.html b/euscanwww/djeuscan/templates/euscan/accounts/herds.html
index d77aa03..d55c034 100644
--- a/euscanwww/djeuscan/templates/euscan/accounts/herds.html
+++ b/euscanwww/djeuscan/templates/euscan/accounts/herds.html
@@ -1,11 +1,14 @@
-{% extends "_base.html" %}
+{% extends "euscan/_datatable.html" %}
 {% load url from future %}
+{% load euscan %}
 
 {% block title %}
-  {{ block.super }} - My herds
+  {{ block.super }} - Watched herds
 {% endblock %}
 
 
 {% block content %}
-  <h2>Welcome {{ user }}</h2>
+  <h2>Watched herds</h2>
+
+  {% herds_table herds %}
 {% endblock %}
diff --git a/euscanwww/djeuscan/templates/euscan/accounts/maintainers.html b/euscanwww/djeuscan/templates/euscan/accounts/maintainers.html
index 1a907b9..06cfc1f 100644
--- a/euscanwww/djeuscan/templates/euscan/accounts/maintainers.html
+++ b/euscanwww/djeuscan/templates/euscan/accounts/maintainers.html
@@ -1,11 +1,14 @@
-{% extends "_base.html" %}
+{% extends "euscan/_datatable.html" %}
 {% load url from future %}
+{% load euscan %}
 
 {% block title %}
-  {{ block.super }} - My maintainer profiles
+  {{ block.super }} - Watched maintainers
 {% endblock %}
 
 
 {% block content %}
-  <h2>Welcome {{ user }}</h2>
+  <h2>Watched maintainers</h2>
+
+  {% maintainers_table maintainers %}
 {% endblock %}
diff --git a/euscanwww/djeuscan/templates/euscan/accounts/packages.html b/euscanwww/djeuscan/templates/euscan/accounts/packages.html
index 7c847f6..bd87b5d 100644
--- a/euscanwww/djeuscan/templates/euscan/accounts/packages.html
+++ b/euscanwww/djeuscan/templates/euscan/accounts/packages.html
@@ -1,15 +1,15 @@
 {% extends "euscan/_datatable.html" %}
 {% load url from future %}
 
-{% load packages %}
+{% load euscan %}
 
 {% block title %}
-  {{ block.super }} - My packages
+  {{ block.super }} - Watched packages
 {% endblock %}
 
 
 {% block content %}
-  <h2>My Packages</h2>
+  <h2>Watched packages</h2>
 
   {% packages packages %}
 
diff --git a/euscanwww/djeuscan/templates/euscan/categories.html b/euscanwww/djeuscan/templates/euscan/categories.html
index 9b0a3eb..1887627 100644
--- a/euscanwww/djeuscan/templates/euscan/categories.html
+++ b/euscanwww/djeuscan/templates/euscan/categories.html
@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 {% load url from future %}
 
 {% block title %}
@@ -10,33 +10,6 @@
 {% block content %}
 <h2>Categories</h2>
 
-<table id="table" class="display">
-  <thead>
-    <th>Category</th>
-    <th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
-    <th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
-    <th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
-    <th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
-    {% if request.GET.extras %}
-    <th>Graphs</th>
-    {% endif %}
-  </thead>
-  <tbody>
-  {% for category in categories %}
-  <tr>
-    <td>
-      <a href="{% url "category" category.category %}">{{ category.category }}</a>
-      {% package_bar category %}
-    </td>
-    {% package_cols category %}
-    {% if request.GET.extras %}
-    <td>
-      <img src="{% url "chart_category" category.category 'packages-monthly-small' %}" />
-      <img src="{% url "chart_category" category.category 'versions-monthly-small' %}" />
-    </td>
-    {% endif %}
-  </tr>
-  {% endfor %}
-  </tbody>
-</table>
+{% categories_table categories request.GET.extras %}
+
 {% endblock %}
diff --git a/euscanwww/djeuscan/templates/euscan/category.html b/euscanwww/djeuscan/templates/euscan/category.html
index 25cf7f7..edfd4c5 100644
--- a/euscanwww/djeuscan/templates/euscan/category.html
+++ b/euscanwww/djeuscan/templates/euscan/category.html
@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 {% load url from future %}
 
 {% block title %}
diff --git a/euscanwww/djeuscan/templates/euscan/herd.html b/euscanwww/djeuscan/templates/euscan/herd.html
index a7470ef..56a7488 100644
--- a/euscanwww/djeuscan/templates/euscan/herd.html
+++ b/euscanwww/djeuscan/templates/euscan/herd.html
@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 {% load url from future %}
 
 {% block title %}
diff --git a/euscanwww/djeuscan/templates/euscan/herds.html b/euscanwww/djeuscan/templates/euscan/herds.html
index 48eb591..ef11932 100644
--- a/euscanwww/djeuscan/templates/euscan/herds.html
+++ b/euscanwww/djeuscan/templates/euscan/herds.html
@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 {% load url from future %}
 
 {% block title %}
@@ -9,35 +9,7 @@
 
 {% block content %}
 <h2>Herds</h2>
-<table id="table" class="display">
-  <thead>
-    <th>Herd</th>
-    <th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
-    <th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
-    <th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
-    <th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
-    {% if request.GET.extras %}
-    <th>Graphs</th>
-    {% endif %}
-  </thead>
-  <tbody>
-  {% for herd in herds %}
-    <tr>
-      <td>
-        <a href="{% url "herd" herd.herds__herd %}">
-          {{ herd.herds__herd }}
-        </a>
-        {% package_bar herd %}
-      </td>
-      {% package_cols herd %}
-      {% if request.GET.extras %}
-        <td>
-          <img src="{% url "chart_herd" herd.herds__herd 'packages-monthly-small' %}" />
-          <img src="{% url "chart_herd" herd.herds__herd 'versions-monthly-small' %}" />
-        </td>
-      {% endif %}
-    </tr>
-  {% endfor %}
-  </tbody>
-</table>
+
+{% herds_table herds request.GET.extras %}
+
 {% endblock %}
diff --git a/euscanwww/djeuscan/templates/euscan/maintainer.html b/euscanwww/djeuscan/templates/euscan/maintainer.html
index 5630209..b4aff14 100644
--- a/euscanwww/djeuscan/templates/euscan/maintainer.html
+++ b/euscanwww/djeuscan/templates/euscan/maintainer.html
@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 {% load url from future %}
 
 {% block title %}
diff --git a/euscanwww/djeuscan/templates/euscan/maintainers.html b/euscanwww/djeuscan/templates/euscan/maintainers.html
index ba0cc32..8c2e6d5 100644
--- a/euscanwww/djeuscan/templates/euscan/maintainers.html
+++ b/euscanwww/djeuscan/templates/euscan/maintainers.html
@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 {% load url from future %}
 
 {% block title %}
@@ -9,41 +9,7 @@
 
 {% block content %}
 <h2>Maintainers</h2>
-<table id="table" class="display">
-  <thead>
-    <th>Maintainer</th>
-    <th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
-    <th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
-    <th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
-    <th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
-    {% if request.GET.extras %}
-      <th>Graphs</th>
-    {% endif %}
-  </thead>
-  <tbody>
-  {% for maintainer in maintainers %}
-    <tr>
-      <td>
-        <a href="{% url "maintainer" maintainer.maintainers__id %}">
-          {% if maintainer.maintainers__name != maintainer.maintainers__email %}
-            {{ maintainer.maintainers__name }} &lt;{{ maintainer.maintainers__email }}&gt;
-          {% else %}
-            {{ maintainer.maintainers__name }}
-          {% endif %}
-        </a>
-        {% package_bar maintainer %}
-      </td>
-      {% package_cols maintainer %}
 
-      {% if request.GET.extras %}
-        <td>
-          <img src="{% url "chart_maintainer" maintainer.maintainers__id 'packages-monthly-small' %}" />
-          <img src="{% url "chart_maintainer" maintainer.maintainers__id 'versions-monthly-small' %}" />
-        </td>
-      {% endif %}
+{% maintainers_table maintainers request.GET.extras %}
 
-    </tr>
-  {% endfor %}
-  </tbody>
-</table>
 {% endblock %}
diff --git a/euscanwww/djeuscan/templates/euscan/overlay.html b/euscanwww/djeuscan/templates/euscan/overlay.html
index b6e105c..c0b2a92 100644
--- a/euscanwww/djeuscan/templates/euscan/overlay.html
+++ b/euscanwww/djeuscan/templates/euscan/overlay.html
@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 
 {% block title %}
 {{ block.super }} - Overlay: {{ overlay }}
diff --git a/euscanwww/djeuscan/templates/euscan/overlays.html b/euscanwww/djeuscan/templates/euscan/overlays.html
index 2efb73f..51c6a7a 100644
--- a/euscanwww/djeuscan/templates/euscan/overlays.html
+++ b/euscanwww/djeuscan/templates/euscan/overlays.html
@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 {% load url from future %}
 
 {% block title %}
diff --git a/euscanwww/djeuscan/templates/euscan/package.html b/euscanwww/djeuscan/templates/euscan/package.html
index 6eb500f..7b3eaec 100644
--- a/euscanwww/djeuscan/templates/euscan/package.html
+++ b/euscanwww/djeuscan/templates/euscan/package.html
@@ -31,9 +31,12 @@
     <button class="refresh-button" data-package="{{ package.category }}/{{ package.name }}">
       Refresh
     </button>
-    <button class="favourite-button" data-url="{% url "favourite_package" package.category package.name %}">
+    <button class="favourite-button {% if favourited %}hide{% endif %}" data-url="{% url "favourite_package" package.category package.name %}">
       Add to favourites
     </button>
+    <button class="unfavourite-button {% if not favourited %}hide{% endif %}" data-url="{% url "unfavourite_package" package.category package.name %}">
+      Remove from favourites
+    </button>
   {% endif %}
 </h2>
 <dl>
@@ -151,7 +154,15 @@
 
   $(".favourite-button").click(function() {
     $.post($(this).data("url"), function() {
-      alert("Favourited!");
+      $(".unfavourite-button").removeClass("hide");
+      $(".favourite-button").addClass("hide");
+    });
+  });
+
+  $(".unfavourite-button").click(function() {
+    $.post($(this).data("url"), function() {
+      $(".favourite-button").removeClass("hide");
+      $(".unfavourite-button").addClass("hide");
     });
   });
 </script>
diff --git a/euscanwww/djeuscan/templates/euscan/world_scan.html b/euscanwww/djeuscan/templates/euscan/world_scan.html
index ebb6561..b16a850 100644
--- a/euscanwww/djeuscan/templates/euscan/world_scan.html
+++ b/euscanwww/djeuscan/templates/euscan/world_scan.html
@@ -1,6 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
-{% load packages %}
+{% load euscan %}
 
 {% block title %}
 {{ block.super }} - World Scan
diff --git a/euscanwww/djeuscan/templatetags/euscan.py b/euscanwww/djeuscan/templatetags/euscan.py
new file mode 100644
index 0000000..41daf4b
--- /dev/null
+++ b/euscanwww/djeuscan/templatetags/euscan.py
@@ -0,0 +1,49 @@
+from django import template
+from django.conf import settings
+
+register = template.Library()
+
+
+@register.inclusion_tag('euscan/_packages.html', takes_context=True)
+def packages(context, pkgs):
+    context['packages'] = pkgs
+    return context
+
+
+@register.inclusion_tag('euscan/_package_cols.html', takes_context=True)
+def package_cols(context, infos):
+    context['infos'] = infos
+    return context
+
+
+@register.inclusion_tag('euscan/_package_bar.html', takes_context=True)
+def package_bar(context, infos):
+    context['infos'] = infos
+    return context
+
+
+@register.inclusion_tag('euscan/_categories_table.html')
+def categories_table(categories, extras=False):
+    return {
+        "categories": categories,
+        "extras": extras,
+        "STATIC_URL": settings.STATIC_URL,
+    }
+
+
+@register.inclusion_tag('euscan/_herds_table.html')
+def herds_table(herds, extras=False):
+    return {
+        "herds": herds,
+        "extras": extras,
+        "STATIC_URL": settings.STATIC_URL,
+    }
+
+
+@register.inclusion_tag('euscan/_maintainers_table.html')
+def maintainers_table(maintainers, extras=False):
+    return {
+        "maintainers": maintainers,
+        "extras": extras,
+        "STATIC_URL": settings.STATIC_URL,
+    }
diff --git a/euscanwww/djeuscan/templatetags/packages.py b/euscanwww/djeuscan/templatetags/packages.py
deleted file mode 100644
index 91e8af3..0000000
--- a/euscanwww/djeuscan/templatetags/packages.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from django import template
-
-register = template.Library()
-
-
-@register.inclusion_tag('euscan/_packages.html', takes_context=True)
-def packages(context, pkgs):
-    context['packages'] = pkgs
-    return context
-
-
-@register.inclusion_tag('euscan/_package_cols.html', takes_context=True)
-def package_cols(context, infos):
-    context['infos'] = infos
-    return context
-
-
-@register.inclusion_tag('euscan/_package_bar.html', takes_context=True)
-def package_bar(context, infos):
-    context['infos'] = infos
-    return context
diff --git a/euscanwww/djeuscan/urls.py b/euscanwww/djeuscan/urls.py
index 17063b6..2fad421 100644
--- a/euscanwww/djeuscan/urls.py
+++ b/euscanwww/djeuscan/urls.py
@@ -18,6 +18,9 @@ package_patterns = patterns('djeuscan.views',
         'package', name="package"),
     url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/favourite$',
         'favourite_package', name="favourite_package"),
+    url((r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/'
+         'unfavourite$'),
+        'unfavourite_package', name="unfavourite_package"),
 )
 
 categories_patterns = patterns('djeuscan.views',
@@ -29,6 +32,8 @@ categories_patterns = patterns('djeuscan.views',
         'chart_category', name="chart_category"),
     url(r'^(?P<category>[\w+][\w+.-]*)/favourite$',
         'favourite_category', name="favourite_category"),
+    url(r'^(?P<category>[\w+][\w+.-]*)/unfavourite$',
+        'unfavourite_category', name="unfavourite_category"),
     url(r'^$', 'categories', name="categories"),
 )
 
@@ -39,6 +44,8 @@ herds_patterns = patterns('djeuscan.views',
         'chart_herd', name="chart_herd"),
     url(r'^(?P<herd>[\@\{\}\w+.-]*)/favourite$', 'favourite_herd',
         name="favourite_herd"),
+    url(r'^(?P<herd>[\@\{\}\w+.-]*)/unfavourite$', 'unfavourite_herd',
+        name="unfavourite_herd"),
     url(r'^$', 'herds', name="herds"),
 )
 
@@ -50,6 +57,8 @@ maintainers_patterns = patterns('djeuscan.views',
         'chart_maintainer', name="chart_maintainer"),
     url(r'^(?P<maintainer_id>\d+)/favourite$',
         'favourite_maintainer', name="favourite_maintainer"),
+    url(r'^(?P<maintainer_id>\d+)/unfavourite$',
+        'unfavourite_maintainer', name="unfavourite_maintainer"),
     url(r'^$', 'maintainers', name="maintainers"),
 )
 
diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py
index 7a34271..730dcf0 100644
--- a/euscanwww/djeuscan/views.py
+++ b/euscanwww/djeuscan/views.py
@@ -167,13 +167,23 @@ def package(request, category, package):
     except EuscanResult.DoesNotExist:
         last_scan = None
 
+    favourited = False
+    if request.user.is_authenticated():
+        try:
+            PackageAssociation.objects.get(user=request.user, package=package)
+        except PackageAssociation.DoesNotExist:
+            pass
+        else:
+            favourited = True
+
     return {
         'package': package,
         'packaged': packaged,
         'upstream': upstream,
         'log': log,
         'vlog': vlog,
-        'last_scan': last_scan
+        'last_scan': last_scan,
+        'favourited': favourited,
     }
 
 
@@ -299,19 +309,25 @@ def accounts_index(request):
 @login_required
 @render_to('euscan/accounts/categories.html')
 def accounts_categories(request):
-    return {}
+    categories = [obj.category for obj in
+                  CategoryAssociation.objects.filter(user=request.user)]
+    return {"categories": categories}
 
 
 @login_required
 @render_to('euscan/accounts/herds.html')
 def accounts_herds(request):
-    return {}
+    herds = [obj.herd for obj in
+             HerdAssociation.objects.filter(user=request.user)]
+    return {"herds": herds}
 
 
 @login_required
 @render_to('euscan/accounts/maintainers.html')
 def accounts_maintainers(request):
-    return {}
+    maintainers = [obj.maintainer for obj in
+                   MaintainerAssociation.objects.filter(user=request.user)]
+    return {"maintainers": maintainers}
 
 
 @login_required
@@ -333,6 +349,18 @@ def favourite_package(request, category, package):
     return {"success": created}
 
 
+@login_required
+@require_POST
+@ajax_request
+def unfavourite_package(request, category, package):
+    package = get_object_or_404(Package, category=category, name=package)
+    obj = get_object_or_404(
+        PackageAssociation, package=package, user=request.user
+    )
+    obj.delete()
+    return {"success": True}
+
+
 @login_required
 @require_POST
 @ajax_request
@@ -344,6 +372,18 @@ def favourite_herd(request, herd):
     return {"success": created}
 
 
+@login_required
+@require_POST
+@ajax_request
+def unfavourite_herd(request, herd):
+    herd = get_object_or_404(Herd, herd=herd)
+    obj = get_object_or_404(
+        HerdAssociation, herd=herd, user=request.user
+    )
+    obj.delete()
+    return {"success": True}
+
+
 @login_required
 @require_POST
 @ajax_request
@@ -355,6 +395,18 @@ def favourite_maintainer(request, maintainer_id):
     return {"success": created}
 
 
+@login_required
+@require_POST
+@ajax_request
+def unfavourite_maintainer(request, maintainer_id):
+    maintainer = get_object_or_404(Maintainer, pk=maintainer_id)
+    obj = get_object_or_404(
+        MaintainerAssociation, maintainer=maintainer, user=request.user
+    )
+    obj.delete()
+    return {"success": True}
+
+
 @login_required
 @require_POST
 @ajax_request
@@ -368,3 +420,14 @@ def favourite_category(request, category):
         user=request.user, category=category
     )
     return {"success": created}
+
+
+@login_required
+@require_POST
+@ajax_request
+def unfavourite_category(request, category):
+    obj = get_object_or_404(
+        CategoryAssociation, user=request.user, category=category
+    )
+    obj.delete()
+    return {"success": True}
diff --git a/euscanwww/htdocs/css/style.css b/euscanwww/htdocs/css/style.css
index 62d6cda..556f02f 100644
--- a/euscanwww/htdocs/css/style.css
+++ b/euscanwww/htdocs/css/style.css
@@ -119,68 +119,66 @@ a:hover, a:active
 
 /*
 code, pre{
-	background-color:transparent;
-	font-family:"Courier New",Courier,monospace;
-	font-size:small;
+    background-color:transparent;
+    font-family:"Courier New",Courier,monospace;
+    font-size:small;
 }
 */
 
 a{
-	color: #3F4C66;
+    color: #3F4C66;
 }
 
 a:link, a:visited, a:active {
-	color: #3F4C66;
-	text-decoration: none;
+    color: #3F4C66;
+    text-decoration: none;
 }
 a:hover {
-	color: #4C5C7B;
-	text-decoration: underline;
+    color: #4C5C7B;
+    text-decoration: underline;
 }
 
 abbr:hover {
-	text-decoration: underline;
+    text-decoration: underline;
 }
 
 pre{
-	border-left:5px solid;
-	padding:0.5em 1em;
-	margin-left:2em;
+    border-left:5px solid;
+    padding:0.5em 1em;
+    margin-left:2em;
 }
 
 dt {
-	font-weight: bold;
+    font-weight: bold;
 }
 
 dd {
-	border-left: 1px solid #ccc;
-	margin: 5px;
-	padding: 5px;
+    border-left: 1px solid #ccc;
+    margin: 5px;
+    padding: 5px;
 }
 
 
 h1 {
-        color: #000;
-	width: 20em;
-	margin-left: auto;
-	margin-right: auto;
+    color: #000;
+    width: 20em;
+    margin-left: auto;
+    margin-right: auto;
 }
 
 .ok {
-	color:#15B100;
+    color:#15B100;
 }
 
-hr
-{
-  margin: 0.3em 1em 0.3em 1em;
-  height: 1px;
-  border: #bcbcbc dashed;
-  border-width: 0 0 1px 0;
-  }
+hr {
+    margin: 0.3em 1em 0.3em 1em;
+    height: 1px;
+    border: #bcbcbc dashed;
+    border-width: 0 0 1px 0;
+}
 
 
-table
-{
+table {
     max-width: 60em;
     /* width: 50%; */
     border-collapse: collapse;
@@ -191,27 +189,24 @@ table
     text-align: left;
 }
 
-td
-{
-  vertical-align: top;
-  }
+td {
+    vertical-align: top;
+}
 
-th
-{
-  font-size: 0.8em;
-  text-align: center;
-  border-bottom: 3px solid;
- }
+th {
+    font-size: 0.8em;
+    text-align: center;
+    border-bottom: 3px solid;
+}
 
-#footer
-{
+#footer {
     background: #8076A1;
     font-size: 0.8em;
     width: 100%;
     margin: 0;
     margin-top: 10px;
     color: #FFF;
- }
+}
 
 #footer p {
     text-align: right;
@@ -220,33 +215,31 @@ th
 
 .err,.ok,.inf
 {
-  margin: 5px;
-  margin-left: auto;
-  margin-right: auto;
-  width: 70%;
-  font-weight:bold;
-  border: 1px dotted #5682AD;
-  }
+    margin: 5px;
+    margin-left: auto;
+    margin-right: auto;
+    width: 70%;
+    font-weight:bold;
+    border: 1px dotted #5682AD;
+}
 
 .added {
-  color: #262;
+    color: #262;
 }
 
 .removed {
-  color: #F00;
+    color: #F00;
 }
 
-.err
-{
-  border-color: #F00;
-  color: #F00;
- }
+.err {
+    border-color: #F00;
+    color: #F00;
+}
 
-.ok
-{
-  border-color: #262;
-  color: #262;
-  }
+.ok {
+    border-color: #262;
+    color: #262;
+}
 
 .logo {
     float: right;
@@ -284,3 +277,7 @@ th
     max-height: 100pt;
     overflow: auto;
 }
+
+.hide {
+    display: none;
+}

From 11c8422c9b4adc73abd5f825661e71f846a1fe1f Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Thu, 14 Jun 2012 10:21:01 +0200
Subject: [PATCH 24/37] euscanwww: Reset password

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/templates/_base.html           |  2 +-
 .../djeuscan/templates/registration/login.html    | 15 +++++++++++----
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/euscanwww/djeuscan/templates/_base.html b/euscanwww/djeuscan/templates/_base.html
index 54ea007..da71b71 100644
--- a/euscanwww/djeuscan/templates/_base.html
+++ b/euscanwww/djeuscan/templates/_base.html
@@ -50,7 +50,7 @@
                 </ul>
                 <li><a href="{% url "django.contrib.auth.views.logout" %}">Logout</a></li>
               {% else %}
-              <li><a href="{% url "django.contrib.auth.views.login" %}?next={% url "accounts_index" %}">Login</a></li>
+                <li><a href="{% url "django.contrib.auth.views.login" %}?next={% url "accounts_index" %}">Login</a></li>
                 <li><a href="{% url "registration.views.register" %}">Register</a></li>
               {% endif %}
 
diff --git a/euscanwww/djeuscan/templates/registration/login.html b/euscanwww/djeuscan/templates/registration/login.html
index d2e3547..eaa7c54 100644
--- a/euscanwww/djeuscan/templates/registration/login.html
+++ b/euscanwww/djeuscan/templates/registration/login.html
@@ -12,17 +12,24 @@
     {% csrf_token %}
     <table>
       <tr>
-        <td>{{ form.username.label_tag }}</td>
+        <th>{{ form.username.label_tag }}</th>
         <td>{{ form.username }}</td>
       </tr>
       <tr>
-        <td>{{ form.password.label_tag }}</td>
+        <th>{{ form.password.label_tag }}</th>
         <td>{{ form.password }}</td>
       </tr>
+      <tr>
+        <td></td>
+        <td>
+          <input type="submit" value="Login" />
+        </td>
+      </tr>
     </table>
-
-    <input type="submit" value="login" />
     <input type="hidden" name="next" value="/" />
   </form>
 
+  <a href="{% url "registration.views.register" %}">Register</a>
+  <a href="{% url "django.contrib.auth.views.password_reset" %}">Forgot password</a>
+
 {% endblock %}

From 9bfd728d4fcfb0a2ed6d92ef63376d5db045e4a0 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Fri, 22 Jun 2012 09:32:39 +0200
Subject: [PATCH 25/37] euscanwww: Moved business logic to djeuscan.processing

Moved stuff from management commands to djeuscan.processing

Signed-off-by: volpino <fox91@anche.no>
---
 .../management/commands/regen_rrds.py         |  22 +-
 .../management/commands/scan_metadata.py      | 153 +--------
 .../management/commands/scan_portage.py       | 305 +----------------
 .../management/commands/scan_upstream.py      | 194 +----------
 .../management/commands/update_counters.py    | 186 +---------
 euscanwww/djeuscan/processing/__init__.py     |   3 +
 euscanwww/djeuscan/processing/regen_rrds.py   |  31 ++
 .../djeuscan/processing/scan_metadata.py      | 148 ++++++++
 euscanwww/djeuscan/processing/scan_portage.py | 320 ++++++++++++++++++
 .../djeuscan/processing/scan_upstream.py      | 139 ++++++++
 .../djeuscan/processing/update_counters.py    | 185 ++++++++++
 .../processing/update_portage_trees.py        |  66 ++++
 euscanwww/djeuscan/tasks.py                   | 298 +++++++++-------
 13 files changed, 1110 insertions(+), 940 deletions(-)
 create mode 100644 euscanwww/djeuscan/processing/__init__.py
 create mode 100644 euscanwww/djeuscan/processing/regen_rrds.py
 create mode 100644 euscanwww/djeuscan/processing/scan_metadata.py
 create mode 100644 euscanwww/djeuscan/processing/scan_portage.py
 create mode 100644 euscanwww/djeuscan/processing/scan_upstream.py
 create mode 100644 euscanwww/djeuscan/processing/update_counters.py
 create mode 100644 euscanwww/djeuscan/processing/update_portage_trees.py

diff --git a/euscanwww/djeuscan/management/commands/regen_rrds.py b/euscanwww/djeuscan/management/commands/regen_rrds.py
index 1c041fa..f6b2ba0 100644
--- a/euscanwww/djeuscan/management/commands/regen_rrds.py
+++ b/euscanwww/djeuscan/management/commands/regen_rrds.py
@@ -1,25 +1,5 @@
 from django.core.management.base import BaseCommand
-from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
-from djeuscan import charts
-
-
-def regen_rrds():
-    """
-    Regenerates the rrd database
-    """
-    for wlog in WorldLog.objects.all():
-        charts.rrd_update('world', wlog.datetime, wlog)
-
-    for clog in CategoryLog.objects.all():
-        charts.rrd_update('category-%s' % clog.category,
-                          clog.datetime, clog)
-
-    for hlog in HerdLog.objects.all():
-        charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
-
-    for mlog in MaintainerLog.objects.all():
-        charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
-                          mlog.datetime, mlog)
+from djeuscan.processing.regen_rrds import regen_rrds
 
 
 class Command(BaseCommand):
diff --git a/euscanwww/djeuscan/management/commands/scan_metadata.py b/euscanwww/djeuscan/management/commands/scan_metadata.py
index 1e2cfdc..5a19315 100644
--- a/euscanwww/djeuscan/management/commands/scan_metadata.py
+++ b/euscanwww/djeuscan/management/commands/scan_metadata.py
@@ -1,142 +1,9 @@
 import sys
-
 from optparse import make_option
 
-from gentoolkit.query import Query
-from gentoolkit.errors import GentoolkitFatalError
-
-from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand
-from django.core.management.color import color_style
-from django.core.exceptions import ValidationError
 
-from djeuscan.models import Package, Herd, Maintainer
-
-
-class ScanMetadata(object):
-    def __init__(self, quiet=False):
-        self.quiet = quiet
-        self.style = color_style()
-
-    @commit_on_success
-    def scan(self, query=None, obj=None):
-        matches = Query(query).find(
-            include_masked=True,
-            in_installed=False,
-        )
-
-        if not matches:
-            sys.stderr.write(
-                self.style.ERROR("Unknown package '%s'\n" % query)
-            )
-            return
-
-        matches = sorted(matches)
-        pkg = matches.pop()
-        if '9999' in pkg.version and len(matches):
-            pkg = matches.pop()
-
-        if not obj:
-            obj, created = Package.objects.get_or_create(
-                category=pkg.category, name=pkg.name
-            )
-        else:
-            created = False
-
-        try:
-            obj.homepage = pkg.environment("HOMEPAGE")
-            obj.description = pkg.environment("DESCRIPTION")
-        except GentoolkitFatalError, err:
-            sys.stderr.write(
-                self.style.ERROR(
-                    "Gentoolkit fatal error: '%s'\n" % str(err)
-                )
-            )
-
-        if created and not self.quiet:
-            sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
-
-        if pkg.metadata:
-            herds = dict(
-                [(herd[0], herd) for herd in pkg.metadata.herds(True)]
-            )
-            maintainers = dict(
-                [(m.email, m) for m in pkg.metadata.maintainers()]
-            )
-
-            existing_herds = [h.herd for h in obj.herds.all()]
-            new_herds = set(herds.keys()).difference(existing_herds)
-            old_herds = set(existing_herds).difference(herds.keys())
-
-            existing_maintainers = [m.email for m in obj.maintainers.all()]
-            new_maintainers = set(
-                maintainers.keys()).difference(existing_maintainers
-            )
-            old_maintainers = set(
-                existing_maintainers).difference(maintainers.keys()
-            )
-
-            for herd in obj.herds.all():
-                if herd.herd in old_herds:
-                    obj.herds.remove(herd)
-
-            for herd in new_herds:
-                herd = self.store_herd(*herds[herd])
-                obj.herds.add(herd)
-
-            for maintainer in obj.maintainers.all():
-                if maintainer.email in old_maintainers:
-                    obj.maintainers.remove(maintainer)
-
-            for maintainer in new_maintainers:
-                maintainer = maintainers[maintainer]
-                try:
-                    maintainer = self.store_maintainer(
-                        maintainer.name, maintainer.email
-                    )
-                    obj.maintainers.add(maintainer)
-                except ValidationError:
-                    sys.stderr.write(
-                        self.style.ERROR("Bad maintainer: '%s' '%s'\n" % (maintainer.name, maintainer.email))
-                    )
-
-        obj.save()
-
-    def store_herd(self, name, email):
-        if not name:
-            name = '{nil}'
-        name = name.strip("\r").strip("\n").strip("\t").strip()
-
-        herd, created = Herd.objects.get_or_create(
-            herd=name,
-            defaults={"email": email}
-        )
-
-        if created and not self.quiet:
-            sys.stdout.write('+ [h] %s <%s>\n' % (name, email))
-
-        herd.email = email
-        herd.save()
-
-        return herd
-
-    def store_maintainer(self, name, email):
-        if not name:
-            name = email
-        if not name:
-            name = '{nil}'
-
-        maintainer, created = Maintainer.objects.get_or_create(
-            email=email,
-            defaults={"name": name}
-        )
-
-        if created:
-            if not self.quiet:
-                sys.stdout.write(
-                    '+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
-                )
-        return maintainer
+from djeuscan.processing.scan_metadata import scan_metadata
 
 
 class Command(BaseCommand):
@@ -158,16 +25,12 @@ class Command(BaseCommand):
     help = 'Scans metadata and fills database'
 
     def handle(self, *args, **options):
-        self.options = options
-
-        scan_metadata = ScanMetadata(quiet=options["quiet"])
-
         if options['all']:
-            for pkg in Package.objects.all():
-                scan_metadata.scan('%s/%s' % (pkg.category, pkg.name), pkg)
-        elif len(args) > 0:
-            for package in args:
-                scan_metadata.scan(package)
+            packages = None
+
+        elif len(args):
+            packages = [pkg for pkg in args]
         else:
-            for package in sys.stdin.readlines():
-                scan_metadata.scan(package[:-1])
+            packages = [pkg[:-1] for pkg in sys.stdin.readlines()]
+
+        scan_metadata(packages=packages, quiet=options["quiet"])
diff --git a/euscanwww/djeuscan/management/commands/scan_portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py
index 039ed78..2bfede8 100644
--- a/euscanwww/djeuscan/management/commands/scan_portage.py
+++ b/euscanwww/djeuscan/management/commands/scan_portage.py
@@ -1,272 +1,8 @@
-import subprocess
-import portage
 import sys
-import os
-import re
 from optparse import make_option
-from collections import defaultdict
 
-from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand
-from django.core.management.color import color_style
-
-from djeuscan.models import Package, Version, VersionLog
-
-
-class ScanPortage(object):
-    def __init__(self, stdout=None, options=None):
-        self.stdout = sys.stdout if stdout is None else stdout
-        self.options = defaultdict(lambda: None) \
-                       if options is None else options
-
-        self.style = color_style()
-        self._cache = {'packages': {}, 'versions': {}}
-        self._overlays = None
-
-    def cache_hash_package(self, category, name):
-        return '%s/%s' % (category, name)
-
-    def cache_store_package(self, package):
-        key = self.cache_hash_package(package.category, package.name)
-        self._cache['packages'][key] = package
-
-    def cache_get_package(self, category, name):
-        return self._cache['packages'].get(
-            self.cache_hash_package(category, name)
-        )
-
-    def cache_hash_version(self, category, name, version, revision, slot,
-                           overlay):
-        key = '%s/%s-%s-r%s %s %s' % (category, name,
-                                      version, revision,
-                                      slot, overlay)
-        return key
-
-    def cache_get_version(self, category, name, version, revision, slot,
-                          overlay):
-        key = self.cache_hash_version(category, name, version, revision, slot,
-                                      overlay)
-        return self._cache['versions'].get(key)
-
-    def cache_store_version(self, version):
-        key = self.cache_hash_version(
-            version.package.category, version.package.name, version.version,
-            version.revision, version.slot, version.overlay
-        )
-        self._cache['versions'][key] = version
-
-    def overlays(self):
-        if self._overlays:
-            return self._overlays
-
-        env = os.environ
-        env['OVERLAYS_LIST'] = 'all'
-        env['PRINT_COUNT_ALWAYS'] = 'never'
-
-        cmd = ['eix', '-!']
-
-        output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
-            communicate()[0]
-        output = output.strip().strip('\n').split('\n')
-
-        overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
-
-        self._overlays = {}
-
-        for line in output:
-            match = overlay_re.match(line)
-            if not match:
-                continue
-            self._overlays[match.group('key')] = match.group('name')
-
-        return self._overlays
-
-    @commit_on_success
-    def scan(self, query=None):
-        env = os.environ
-        env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
-
-        cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages',
-               '-x']
-        if query:
-            cmd.extend(['--exact', query])
-
-        if self.options['all']:
-            if not self.options['quiet']:
-                self.stdout.write('Killing existing versions...')
-                self.stdout.flush()
-            Version.objects.filter(packaged=True).update(alive=False)
-            if not self.options['quiet']:
-                self.stdout.write('done\n')
-
-        output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
-            communicate()[0]
-        output = output.strip().strip('\n')
-
-        if len(output) == 0:
-            if not query:
-                return
-            if self.options['purge-packages']:
-                if not self.options['quiet']:
-                    sys.stdout.write('- [p] %s\n' % (query))
-                if '/' in query:
-                    cat, pkg = portage.catsplit(query)
-                    Package.objects.filter(category=cat, name=pkg).delete()
-                else:
-                    Package.objects.filter(name=query).delete()
-            else:
-                sys.stderr.write(
-                    self.style.ERROR(
-                        "Unknown package '%s'\n" % query
-                    )
-                )
-            return
-
-        output = output.split('\n')
-        packages = {}
-
-        line_re = re.compile(
-            r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$'
-        )
-
-        package = None
-
-        for line in output:
-            match = line_re.match(line)
-
-            if not match:
-                continue
-
-            cpv = match.group('cpv')
-            slot = match.group('slot')
-            overlay = match.group('overlay')
-
-            cat, pkg, ver, rev = portage.catpkgsplit(cpv)
-
-            packages['%s/%s' % (cat, pkg)] = True
-
-            if not package or not \
-               (cat == package.category and pkg == package.name):
-                package = self.store_package(cat, pkg)
-
-            self.store_version(package, cpv, slot, overlay)
-
-        if self.options['purge-packages'] and not query:
-            for package in Package.objects.all():
-                cp = "%s/%s" % (package.category, package.name)
-                if cp not in packages:
-                    if not self.options['quiet']:
-                        sys.stdout.write('- [p] %s\n' % (package))
-                    package.delete()
-
-    def store_package(self, cat, pkg):
-        created = False
-        obj = self.cache_get_package(cat, pkg)
-
-        if not obj:
-            obj, created = Package.objects.get_or_create(
-                category=cat,
-                name=pkg
-            )
-            self.cache_store_package(obj)
-
-        if created:
-            if not self.options['quiet']:
-                sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
-
-        # Set all versions dead, then set found versions alive and
-        # delete old versions
-        if not self.options['all']:
-            Version.objects.filter(
-                package=obj,
-                packaged=True
-            ).update(alive=False)
-
-        return obj
-
-    def store_version(self, package, cpv, slot, overlay):
-        cat, pkg, ver, rev = portage.catpkgsplit(cpv)
-
-        overlays = self.overlays()
-
-        if overlay in overlays:
-            overlay = overlays[overlay]
-        else:
-            overlay = 'gentoo'
-
-        created = False
-        obj = self.cache_get_version(
-            package.category, package.name, ver, rev, slot, overlay
-        )
-        if not obj:
-            obj, created = Version.objects.get_or_create(
-                package=package, slot=slot,
-                revision=rev, version=ver,
-                overlay=overlay,
-                defaults={"alive": True, "packaged": True}
-            )
-        if not created: # Created objects have defaults values
-            obj.alive = True
-            obj.packaged = True
-            obj.save()
-
-        if created:
-            self.cache_store_version(obj)
-
-        # nothing to do (note: it can't be an upstream version because
-        # overlay can't be empty here)
-        if not created:
-            return
-
-        if not self.options['quiet']:
-            sys.stdout.write('+ [v] %s \n' % (obj))
-
-        if overlay == 'gentoo':
-            package.n_packaged += 1
-        else:
-            package.n_overlay += 1
-        package.n_versions += 1
-        package.save()
-
-        if self.options['no-log']:
-            return
-
-        VersionLog.objects.create(
-            package=obj.package,
-            action=VersionLog.VERSION_ADDED,
-            slot=obj.slot,
-            revision=obj.revision,
-            version=obj.version,
-            overlay=obj.overlay
-        )
-
-@commit_on_success
-def purge_versions(quiet=False, nolog=False):
-    # For each dead versions
-    for version in Version.objects.filter(packaged=True, alive=False):
-        if version.overlay == 'gentoo':
-            version.package.n_packaged -= 1
-        else:
-            version.package.n_overlay -= 1
-        version.package.n_versions -= 1
-        version.package.save()
-
-        if not quiet:
-            sys.stdout.write('- [v] %s\n' % (version))
-
-        if nolog:
-            continue
-
-        VersionLog.objects.create(
-            package=version.package,
-            action=VersionLog.VERSION_REMOVED,
-            slot=version.slot,
-            revision=version.revision,
-            version=version.version,
-            overlay=version.overlay
-        )
-
-    Version.objects.filter(packaged=True, alive=False).delete()
+from djeuscan.processing.scan_portage import scan_portage
 
 
 class Command(BaseCommand):
@@ -309,33 +45,20 @@ class Command(BaseCommand):
     help = 'Scans portage tree and fills database'
 
     def handle(self, *args, **options):
-        scan_portage = ScanPortage(stdout=self.stdout, options=options)
-
-        if not options['quiet']:
-            self.stdout.write('Scanning portage tree...\n')
-
-        if options['prefetch']:
-            if not options['quiet']:
-                self.stdout.write('Prefetching objects...')
-                self.stdout.flush()
-            for package in Package.objects.all():
-                scan_portage.cache_store_package(package)
-            for version in Version.objects.select_related('package').all():
-                scan_portage.cache_store_version(version)
-            if not options['quiet']:
-                self.stdout.write('done\n')
-
         if options['all']:
-            scan_portage.scan()
+            packages = None
+
         elif len(args):
-            for package in args:
-                scan_portage.scan(package)
+            packages = [pkg for pkg in args]
         else:
-            for package in sys.stdin.readlines():
-                scan_portage.scan(package[:-1])
+            packages = [pkg[:-1] for pkg in sys.stdin.readlines()]
 
-        if options['purge-versions']:
-            purge_versions(options["quiet"], options["no-log"])
-
-        if not options['quiet']:
-            self.stdout.write('Done.\n')
+        scan_portage(
+            packages=packages,
+            no_log=options["no-log"],
+            purge_packages=options["purge-packages"],
+            purge_versions=options["purge-versions"],
+            prefetch=options["prefetch"],
+            quiet=options["quiet"],
+            stdout=self.stdout,
+        )
diff --git a/euscanwww/djeuscan/management/commands/scan_upstream.py b/euscanwww/djeuscan/management/commands/scan_upstream.py
index f850e52..043fb83 100644
--- a/euscanwww/djeuscan/management/commands/scan_upstream.py
+++ b/euscanwww/djeuscan/management/commands/scan_upstream.py
@@ -1,162 +1,9 @@
-import portage
 import sys
-import re
-
 from optparse import make_option
 
-from django.utils import timezone
-from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand
 
-from euscan import CONFIG, output
-from euscan.scan import scan_upstream
-
-from djeuscan.models import Package, Version, EuscanResult, VersionLog
-
-
-class ScanUpstream(object):
-    def __init__(self, quiet=False):
-        self.quiet = quiet
-
-    def scan(self, package):
-        CONFIG["format"] = "dict"
-        output.set_query(package)
-
-        scan_upstream(package)
-
-        out = output.get_formatted_output()
-        out_json = output.get_formatted_output("json")
-
-        try:
-            cpv = out[package]["metadata"]["cpv"]
-        except KeyError:
-            return {}
-
-        obj = self.store_package(cpv)
-
-        for res in out[package]["result"]:
-            self.store_version(obj, res["version"], " ".join(res["urls"]))
-
-        self.store_result(obj, out_json)
-
-        return out
-
-    def store_result(self, package, log):
-        # Remove previous logs
-        EuscanResult.objects.filter(package=package).delete()
-
-        obj = EuscanResult()
-        obj.package = package
-        obj.result = log
-        obj.datetime = timezone.now()
-        obj.save()
-
-    def store_package(self, cpv):
-        cat, pkg, ver, rev = portage.catpkgsplit(cpv)
-
-        obj, created = Package.objects.get_or_create(category=cat, name=pkg)
-
-        if created and not self.quiet:
-            sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
-
-        # Set all versions dead, then set found versions alive and
-        # delete old versions
-        Version.objects.filter(package=obj, packaged=False).update(alive=False)
-
-        return obj
-
-    def store_version(self, package, ver, url):
-        obj, created = Version.objects.get_or_create(
-            package=package, slot='', revision='r0', version=ver, overlay='',
-            defaults={"alive": True, "urls": url, "packaged": False}
-        )
-        if not created:
-            obj.alive = True
-            obj.urls = url
-            obj.packaged = False
-            obj.save()
-
-        # If it's not a new version, just update the object and continue
-        if not created:
-            return
-
-        if not self.quiet:
-            sys.stdout.write('+ [u] %s %s\n' % (obj, url))
-
-        VersionLog.objects.create(
-            package=package,
-            action=VersionLog.VERSION_ADDED,
-            slot='',
-            revision='r0',
-            version=ver,
-            overlay=''
-        )
-
-        package.n_versions += 1
-        package.save()
-
-    @commit_on_success
-    def parse_output(self, output):
-        from portage.versions import _cp
-
-        if type(_cp) == dict:
-            _cp = _cp["dots_allowed_in_PN"]
-
-        package_re = re.compile(
-            r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$'
-        )
-        version_re = re.compile(
-            r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$'
-        )
-
-        package = None
-        log = ""
-
-        while True:
-            line = output.readline()
-            if line == '':
-                break
-            match = package_re.match(line)
-            if match:
-                if package:
-                    self.store_result(package, log)
-
-                cpv = match.group('cpv')
-                package = self.store_package(cpv)
-                log = line
-                continue
-
-            log += line
-
-            match = version_re.match(line)
-            if match:
-                ver = match.group('ver')
-                url = match.group('url')
-                self.store_version(package, ver, url)
-
-        if package:
-            self.store_result(package, log)
-
-
-@commit_on_success
-def purge_versions(quiet=False):
-    # For each dead versions
-    for version in Version.objects.filter(packaged=False, alive=False):
-        VersionLog.objects.create(
-            package=version.package,
-            action=VersionLog.VERSION_REMOVED,
-            slot=version.slot,
-            revision=version.revision,
-            version=version.version,
-            overlay=version.overlay
-        )
-
-        version.package.n_versions -= 1
-        version.package.save()
-
-        if not quiet:
-            sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
-    Version.objects.filter(packaged=False, alive=False).delete()
+from djeuscan.processing.scan_upstream import scan_upstream
 
 
 class Command(BaseCommand):
@@ -168,11 +15,6 @@ class Command(BaseCommand):
             dest='all',
             default=False,
             help='Scan all packages'),
-        make_option('--feed',
-            action='store_true',
-            dest='feed',
-            default=False,
-            help='Read euscan output from stdin'),
         make_option('--purge-versions',
             action='store_true',
             dest='purge-versions',
@@ -188,29 +30,17 @@ class Command(BaseCommand):
     help = 'Scans metadata and fills database'
 
     def handle(self, *args, **options):
-        scan_upstream = ScanUpstream(options["quiet"])
-
-        if options['feed']:
-            scan_upstream.parse_output(sys.stdin)
-            if options['purge-versions']:
-                purge_versions(options["quiet"])
-            return
-
-        if not options['quiet']:
-            self.stdout.write('Scanning upstream...\n')
-
         if options['all']:
-            for pkg in Package.objects.all():
-                scan_upstream.scan('%s/%s' % (pkg.category, pkg.name))
-        elif args:
-            for arg in args:
-                scan_upstream.scan(arg)
+            packages = None
+
+        elif len(args):
+            packages = [pkg for pkg in args]
         else:
-            for package in sys.stdin.readlines():
-                scan_upstream.scan(package[:-1])
+            packages = [pkg[:-1] for pkg in sys.stdin.readlines()]
 
-        if options['purge-versions']:
-            purge_versions(options["quiet"])
-
-        if not options['quiet']:
-            self.stdout.write('Done.\n')
+        scan_upstream(
+            packages=packages,
+            purge_versions=options["purge-versions"],
+            quiet=options["quiet"],
+            stdout=self.stdout,
+        )
diff --git a/euscanwww/djeuscan/management/commands/update_counters.py b/euscanwww/djeuscan/management/commands/update_counters.py
index 0b67a39..7dc407e 100644
--- a/euscanwww/djeuscan/management/commands/update_counters.py
+++ b/euscanwww/djeuscan/management/commands/update_counters.py
@@ -1,185 +1,8 @@
-import sys
 from optparse import make_option
 
-from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand
-from django.utils import timezone
 
-from djeuscan.models import Package, Herd, Maintainer, Version
-from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
-from djeuscan import charts
-
-from distutils.version import StrictVersion, LooseVersion
-
-
-def compare_versions(version1, version2):
-    try:
-        return cmp(StrictVersion(version1), StrictVersion(version2))
-    # in case of abnormal version number, fall back to LooseVersion
-    except ValueError:
-        return cmp(LooseVersion(version1), LooseVersion(version2))
-
-
-def add_safe(storage, key):
-    if key not in storage:
-        storage[key] = 1
-    else:
-        storage[key] += 1
-
-
-def add_last_ver(storage, version):
-    key = version['package_id']
-    if key not in storage:
-        storage[key] = version
-        return
-    if version['version'].startswith('9999'):
-        return
-    if compare_versions(storage[key]['version'],
-                        version['version']) < 0:
-        storage[key] = version
-
-
-@commit_on_success
-def update_counters(stdout=None, **options):
-    if stdout is None:
-        stdout = sys.stdout
-
-    now = timezone.now()
-
-    categories = {}
-    herds = {}
-    maintainers = {}
-
-    wlog = None
-
-    if not options['nolog']:
-        wlog = WorldLog()
-        wlog.datetime = now
-
-        for cat in Package.objects.values('category').distinct():
-            clog = CategoryLog()
-            clog.datetime = now
-            clog.category = cat['category']
-            categories[clog.category] = clog
-
-        for herd in Herd.objects.all():
-            hlog = HerdLog()
-            hlog.datetime = now
-            hlog.herd = herd
-            herds[herd.id] = hlog
-
-        for maintainer in Maintainer.objects.all():
-            mlog = MaintainerLog()
-            mlog.datetime = now
-            mlog.maintainer = maintainer
-            maintainers[maintainer.id] = mlog
-
-    package_queryset = Package.objects.all()
-
-    n_versions = {}
-    n_packaged = {}
-    n_overlay = {}
-
-    last_versions_gentoo = {}
-    last_versions_overlay = {}
-    last_versions_upstream = {}
-
-    if not options['fast']:
-        attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
-        for version in Version.objects.all().values(*attrs):
-            overlay, packaged = version['overlay'], version['packaged']
-            package_id = version['package_id']
-
-            add_safe(n_versions, package_id)
-
-            if not packaged:
-                add_last_ver(last_versions_upstream, version)
-                continue
-            if overlay == 'gentoo':
-                add_safe(n_packaged, package_id)
-                add_last_ver(last_versions_gentoo, version)
-            else:
-                add_safe(n_overlay, package_id)
-                add_last_ver(last_versions_overlay, version)
-
-    for package in package_queryset.select_related('herds', 'maintainers'):
-        if not options['fast']:
-            package.n_versions = n_versions.get(package.id, 0)
-            package.n_packaged = n_packaged.get(package.id, 0)
-            package.n_overlay = n_overlay.get(package.id, 0)
-
-            default = {'id': None}
-            package.last_version_gentoo_id = last_versions_gentoo.get(
-                package.id, default
-            )['id']
-            package.last_version_overlay_id = last_versions_overlay.get(
-                package.id, default
-            )['id']
-            package.last_version_upstream_id = last_versions_upstream.get(
-                package.id, default
-            )['id']
-
-            package.save()
-
-        n_packages_gentoo = int(package.n_packaged == package.n_versions)
-        n_packages_overlay = int(package.n_overlay and package.n_packaged \
-                                 + package.n_overlay == package.n_versions)
-        n_packages_outdated = int(package.n_packaged + package.n_overlay \
-                                  < package.n_versions)
-
-        def update_row(storage, key):
-            storage[key].n_packages_gentoo += n_packages_gentoo
-            storage[key].n_packages_overlay += n_packages_overlay
-            storage[key].n_packages_outdated += n_packages_outdated
-
-            storage[key].n_versions_gentoo += package.n_packaged
-            storage[key].n_versions_overlay += package.n_overlay
-            storage[key].n_versions_upstream += package.n_versions - \
-                                                package.n_packaged - \
-                                                package.n_overlay
-
-        def update_log(storage, qs):
-            for row in qs:
-                update_row(storage, row['id'])
-
-        if not options['nolog']:
-            update_log(herds, package.herds.all().values('id'))
-            update_log(maintainers, package.maintainers.all().values('id'))
-            update_row(categories, package.category)
-
-            wlog.n_packages_gentoo += n_packages_gentoo
-            wlog.n_packages_overlay += n_packages_overlay
-            wlog.n_packages_outdated += n_packages_outdated
-
-            wlog.n_versions_gentoo += package.n_packaged
-            wlog.n_versions_overlay += package.n_overlay
-            wlog.n_versions_upstream += package.n_versions - \
-                                        package.n_packaged - \
-                                        package.n_overlay
-
-    if options['nolog']:
-        return
-
-    for clog in categories.values():
-        if not options['quiet']:
-            stdout.write('+ [cl] %s\n' % clog)
-        charts.rrd_update('category-%s' % clog.category, now, clog)
-        clog.save()
-
-    for hlog in herds.values():
-        if not options['quiet']:
-            stdout.write('+ [hl] %s\n' % hlog)
-        charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
-        hlog.save()
-
-    for mlog in maintainers.values():
-        if not options['quiet']:
-            stdout.write('+ [ml] %s\n' % mlog)
-        charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
-        mlog.save()
-
-    charts.rrd_update('world', now, wlog)
-    wlog.save()
+from djeuscan.processing.update_counters import update_counters
 
 
 class Command(BaseCommand):
@@ -205,4 +28,9 @@ class Command(BaseCommand):
     )
 
     def handle(self, *args, **options):
-        update_counters(stdout=self.stdout, **options)
+        update_counters(
+            stdout=self.stdout,
+            fast=options["fast"],
+            quiet=options["quiet"],
+            nolog=options["nolog"],
+        )
diff --git a/euscanwww/djeuscan/processing/__init__.py b/euscanwww/djeuscan/processing/__init__.py
new file mode 100644
index 0000000..c4ea508
--- /dev/null
+++ b/euscanwww/djeuscan/processing/__init__.py
@@ -0,0 +1,3 @@
+class FakeLogger(object):
+    def __getattr__(self, key):
+        return lambda *x, **y: None
diff --git a/euscanwww/djeuscan/processing/regen_rrds.py b/euscanwww/djeuscan/processing/regen_rrds.py
new file mode 100644
index 0000000..f578aa7
--- /dev/null
+++ b/euscanwww/djeuscan/processing/regen_rrds.py
@@ -0,0 +1,31 @@
+from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
+from djeuscan import charts
+
+from djeuscan.processing import FakeLogger
+
+
+def regen_rrds(logger=None):
+    """
+    Regenerates the rrd database
+    """
+
+    if logger is None:
+        logger = FakeLogger()
+
+    logger.info("Regenering RRDs for world")
+    for wlog in WorldLog.objects.all():
+        charts.rrd_update('world', wlog.datetime, wlog)
+
+    logger.info("Regenering RRDs for categories")
+    for clog in CategoryLog.objects.all():
+        charts.rrd_update('category-%s' % clog.category,
+                          clog.datetime, clog)
+
+    logger.info("Regenering RRDs for herds")
+    for hlog in HerdLog.objects.all():
+        charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
+
+    logger.info("Regenering RRDs for maintainers")
+    for mlog in MaintainerLog.objects.all():
+        charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
+                          mlog.datetime, mlog)
diff --git a/euscanwww/djeuscan/processing/scan_metadata.py b/euscanwww/djeuscan/processing/scan_metadata.py
new file mode 100644
index 0000000..15ab108
--- /dev/null
+++ b/euscanwww/djeuscan/processing/scan_metadata.py
@@ -0,0 +1,148 @@
+import sys
+from gentoolkit.query import Query
+from gentoolkit.errors import GentoolkitFatalError
+
+from django.db.transaction import commit_on_success
+from django.core.management.color import color_style
+from django.core.exceptions import ValidationError
+
+from djeuscan.models import Package, Herd, Maintainer
+
+
+class ScanMetadata(object):
+    def __init__(self, quiet=False):
+        self.quiet = quiet
+        self.style = color_style()
+
+    @commit_on_success
+    def scan(self, query=None, obj=None):
+        matches = Query(query).find(
+            include_masked=True,
+            in_installed=False,
+        )
+
+        if not matches:
+            sys.stderr.write(
+                self.style.ERROR("Unknown package '%s'\n" % query)
+            )
+            return
+
+        matches = sorted(matches)
+        pkg = matches.pop()
+        if '9999' in pkg.version and len(matches):
+            pkg = matches.pop()
+
+        if not obj:
+            obj, created = Package.objects.get_or_create(
+                category=pkg.category, name=pkg.name
+            )
+        else:
+            created = False
+
+        try:
+            obj.homepage = pkg.environment("HOMEPAGE")
+            obj.description = pkg.environment("DESCRIPTION")
+        except GentoolkitFatalError, err:
+            sys.stderr.write(
+                self.style.ERROR(
+                    "Gentoolkit fatal error: '%s'\n" % str(err)
+                )
+            )
+
+        if created and not self.quiet:
+            sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
+
+        if pkg.metadata:
+            herds = dict(
+                [(herd[0], herd) for herd in pkg.metadata.herds(True)]
+            )
+            maintainers = dict(
+                [(m.email, m) for m in pkg.metadata.maintainers()]
+            )
+
+            existing_herds = [h.herd for h in obj.herds.all()]
+            new_herds = set(herds.keys()).difference(existing_herds)
+            old_herds = set(existing_herds).difference(herds.keys())
+
+            existing_maintainers = [m.email for m in obj.maintainers.all()]
+            new_maintainers = set(
+                maintainers.keys()).difference(existing_maintainers
+            )
+            old_maintainers = set(
+                existing_maintainers).difference(maintainers.keys()
+            )
+
+            for herd in obj.herds.all():
+                if herd.herd in old_herds:
+                    obj.herds.remove(herd)
+
+            for herd in new_herds:
+                herd = self.store_herd(*herds[herd])
+                obj.herds.add(herd)
+
+            for maintainer in obj.maintainers.all():
+                if maintainer.email in old_maintainers:
+                    obj.maintainers.remove(maintainer)
+
+            for maintainer in new_maintainers:
+                maintainer = maintainers[maintainer]
+                try:
+                    maintainer = self.store_maintainer(
+                        maintainer.name, maintainer.email
+                    )
+                    obj.maintainers.add(maintainer)
+                except ValidationError:
+                    sys.stderr.write(
+                        self.style.ERROR("Bad maintainer: '%s' '%s'\n" % \
+                                         (maintainer.name, maintainer.email))
+                    )
+
+        obj.save()
+
+    def store_herd(self, name, email):
+        if not name:
+            name = '{nil}'
+        name = name.strip("\r").strip("\n").strip("\t").strip()
+
+        herd, created = Herd.objects.get_or_create(
+            herd=name,
+            defaults={"email": email}
+        )
+
+        if created and not self.quiet:
+            sys.stdout.write('+ [h] %s <%s>\n' % (name, email))
+
+        herd.email = email
+        herd.save()
+
+        return herd
+
+    def store_maintainer(self, name, email):
+        if not name:
+            name = email
+        if not name:
+            name = '{nil}'
+
+        maintainer, created = Maintainer.objects.get_or_create(
+            email=email,
+            defaults={"name": name}
+        )
+
+        if created:
+            if not self.quiet:
+                sys.stdout.write(
+                    '+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
+                )
+        return maintainer
+
+
+def scan_metadata(packages=None, quiet=False, logger=None):
+    scan_handler = ScanMetadata(quiet=quiet)
+    if packages is None:
+        packages = Package.objects.all()
+
+    for pkg in packages:
+        if isinstance(pkg, Package):
+            scan_handler.scan('%s/%s' % (pkg.category, pkg.name), pkg)
+        else:
+            scan_handler.scan(pkg)
diff --git a/euscanwww/djeuscan/processing/scan_portage.py b/euscanwww/djeuscan/processing/scan_portage.py
new file mode 100644
index 0000000..43e91f4
--- /dev/null
+++ b/euscanwww/djeuscan/processing/scan_portage.py
@@ -0,0 +1,320 @@
+import subprocess
+import portage
+import sys
+import os
+import re
+
+from django.db.transaction import commit_on_success
+from django.core.management.color import color_style
+
+from djeuscan.models import Package, Version, VersionLog
+
+
+class ScanPortage(object):
+    def __init__(self, stdout=None, no_log=False, purge_packages=False,
+                 purge_versions=False, kill_versions=False, quiet=False):
+        self.stdout = sys.stdout if stdout is None else stdout
+        self.no_log = no_log
+        self.purge_packages = purge_packages
+        self.purge_versions = purge_versions
+        self.kill_versions = kill_versions
+        self.quiet = quiet
+
+        self.style = color_style()
+        self._cache = {'packages': {}, 'versions': {}}
+        self._overlays = None
+
+    def cache_hash_package(self, category, name):
+        return '%s/%s' % (category, name)
+
+    def cache_store_package(self, package):
+        key = self.cache_hash_package(package.category, package.name)
+        self._cache['packages'][key] = package
+
+    def cache_get_package(self, category, name):
+        return self._cache['packages'].get(
+            self.cache_hash_package(category, name)
+        )
+
+    def cache_hash_version(self, category, name, version, revision, slot,
+                           overlay):
+        key = '%s/%s-%s-r%s %s %s' % (category, name,
+                                      version, revision,
+                                      slot, overlay)
+        return key
+
+    def cache_get_version(self, category, name, version, revision, slot,
+                          overlay):
+        key = self.cache_hash_version(category, name, version, revision, slot,
+                                      overlay)
+        return self._cache['versions'].get(key)
+
+    def cache_store_version(self, version):
+        key = self.cache_hash_version(
+            version.package.category, version.package.name, version.version,
+            version.revision, version.slot, version.overlay
+        )
+        self._cache['versions'][key] = version
+
+    def overlays(self):
+        if self._overlays:
+            return self._overlays
+
+        env = os.environ
+        env['OVERLAYS_LIST'] = 'all'
+        env['PRINT_COUNT_ALWAYS'] = 'never'
+
+        cmd = ['eix', '-!']
+
+        output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
+            communicate()[0]
+        output = output.strip().strip('\n').split('\n')
+
+        overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
+
+        self._overlays = {}
+
+        for line in output:
+            match = overlay_re.match(line)
+            if not match:
+                continue
+            self._overlays[match.group('key')] = match.group('name')
+
+        return self._overlays
+
+    @commit_on_success
+    def scan(self, query=None):
+        env = os.environ
+        env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
+
+        cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages',
+               '-x']
+        if query:
+            cmd.extend(['--exact', query])
+
+        if self.kill_versions:
+            if not self.quiet:
+                self.stdout.write('Killing existing versions...')
+                self.stdout.flush()
+            Version.objects.filter(packaged=True).update(alive=False)
+            if not self.quiet:
+                self.stdout.write('done\n')
+
+        output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
+            communicate()[0]
+        output = output.strip().strip('\n')
+
+        if len(output) == 0:
+            if not query:
+                return
+            if self.purge_packages:
+                if not self.quiet:
+                    sys.stdout.write('- [p] %s\n' % (query))
+                if '/' in query:
+                    cat, pkg = portage.catsplit(query)
+                    Package.objects.filter(category=cat, name=pkg).delete()
+                else:
+                    Package.objects.filter(name=query).delete()
+            else:
+                sys.stderr.write(
+                    self.style.ERROR(
+                        "Unknown package '%s'\n" % query
+                    )
+                )
+            return
+
+        output = output.split('\n')
+        packages = {}
+
+        line_re = re.compile(
+            r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$'
+        )
+
+        package = None
+
+        for line in output:
+            match = line_re.match(line)
+
+            if not match:
+                continue
+
+            cpv = match.group('cpv')
+            slot = match.group('slot')
+            overlay = match.group('overlay')
+
+            cat, pkg, ver, rev = portage.catpkgsplit(cpv)
+
+            packages['%s/%s' % (cat, pkg)] = True
+
+            if not package or not \
+               (cat == package.category and pkg == package.name):
+                package = self.store_package(cat, pkg)
+
+            self.store_version(package, cpv, slot, overlay)
+
+        if self.purge_packages and not query:
+            for package in Package.objects.all():
+                cp = "%s/%s" % (package.category, package.name)
+                if cp not in packages:
+                    if not self.quiet:
+                        sys.stdout.write('- [p] %s\n' % (package))
+                    package.delete()
+
+    def store_package(self, cat, pkg):
+        created = False
+        obj = self.cache_get_package(cat, pkg)
+
+        if not obj:
+            obj, created = Package.objects.get_or_create(
+                category=cat,
+                name=pkg
+            )
+            self.cache_store_package(obj)
+
+        if created:
+            if not self.quiet:
+                sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
+
+        # Set all versions dead, then set found versions alive and
+        # delete old versions
+        if not self.kill_versions:
+            Version.objects.filter(
+                package=obj,
+                packaged=True
+            ).update(alive=False)
+
+        return obj
+
+    def store_version(self, package, cpv, slot, overlay):
+        cat, pkg, ver, rev = portage.catpkgsplit(cpv)
+
+        overlays = self.overlays()
+
+        if overlay in overlays:
+            overlay = overlays[overlay]
+        else:
+            overlay = 'gentoo'
+
+        created = False
+        obj = self.cache_get_version(
+            package.category, package.name, ver, rev, slot, overlay
+        )
+        if not obj:
+            obj, created = Version.objects.get_or_create(
+                package=package, slot=slot,
+                revision=rev, version=ver,
+                overlay=overlay,
+                defaults={"alive": True, "packaged": True}
+            )
+        if not created:  # Created objects have defaults values
+            obj.alive = True
+            obj.packaged = True
+            obj.save()
+
+        if created:
+            self.cache_store_version(obj)
+
+        # nothing to do (note: it can't be an upstream version because
+        # overlay can't be empty here)
+        if not created:
+            return
+
+        if not self.quiet:
+            sys.stdout.write('+ [v] %s \n' % (obj))
+
+        if overlay == 'gentoo':
+            package.n_packaged += 1
+        else:
+            package.n_overlay += 1
+        package.n_versions += 1
+        package.save()
+
+        if self.no_log:
+            return
+
+        VersionLog.objects.create(
+            package=obj.package,
+            action=VersionLog.VERSION_ADDED,
+            slot=obj.slot,
+            revision=obj.revision,
+            version=obj.version,
+            overlay=obj.overlay
+        )
+
+
+@commit_on_success
+def purge_versions(quiet=False, nolog=False):
+    # For each dead versions
+    for version in Version.objects.filter(packaged=True, alive=False):
+        if version.overlay == 'gentoo':
+            version.package.n_packaged -= 1
+        else:
+            version.package.n_overlay -= 1
+        version.package.n_versions -= 1
+        version.package.save()
+
+        if not quiet:
+            sys.stdout.write('- [v] %s\n' % (version))
+
+        if nolog:
+            continue
+
+        VersionLog.objects.create(
+            package=version.package,
+            action=VersionLog.VERSION_REMOVED,
+            slot=version.slot,
+            revision=version.revision,
+            version=version.version,
+            overlay=version.overlay
+        )
+
+    Version.objects.filter(packaged=True, alive=False).delete()
+
+
+def scan_portage(packages=None, no_log=False, purge_packages=False,
+                 purge_versions=False, prefetch=False, logger=None,
+                 quiet=False, stdout=None):
+    stdout = sys.stdout if stdout is None else stdout
+
+    kill_versions = False
+    if packages is None:
+        prefetch = True
+        kill_versions = True
+
+    scan_handler = ScanPortage(
+        stdout=stdout,
+        no_log=no_log,
+        purge_packages=purge_packages,
+        purge_versions=purge_versions,
+        kill_versions=kill_versions,
+        quiet=quiet,
+    )
+
+    if not quiet:
+        stdout.write('Scanning portage tree...\n')
+
+    if prefetch:
+        if quiet:
+            stdout.write('Prefetching objects...')
+            stdout.flush()
+        for package in Package.objects.all():
+            scan_handler.cache_store_package(package)
+        for version in Version.objects.select_related('package').all():
+            scan_handler.cache_store_version(version)
+        if quiet:
+            stdout.write('done\n')
+
+    if packages is None:
+        scan_handler.scan()
+    else:
+        for pkg in packages:
+            if isinstance(pkg, Package):
+                scan_handler.scan('%s/%s' % (pkg.category, pkg.name), pkg)
+            else:
+                scan_handler.scan(pkg)
+
+    if purge_versions:
+        purge_versions(quiet, no_log)
+
+    if not quiet:
+        stdout.write('Done.\n')
diff --git a/euscanwww/djeuscan/processing/scan_upstream.py b/euscanwww/djeuscan/processing/scan_upstream.py
new file mode 100644
index 0000000..9eee48b
--- /dev/null
+++ b/euscanwww/djeuscan/processing/scan_upstream.py
@@ -0,0 +1,139 @@
+import portage
+import sys
+
+from django.utils import timezone
+from django.db.transaction import commit_on_success
+
+from euscan import CONFIG, output
+from euscan.scan import scan_upstream as euscan_scan_upstream
+
+from djeuscan.models import Package, Version, EuscanResult, VersionLog
+
+
+class ScanUpstream(object):
+    def __init__(self, quiet=False):
+        self.quiet = quiet
+
+    def scan(self, package):
+        CONFIG["format"] = "dict"
+        output.set_query(package)
+
+        euscan_scan_upstream(package)
+
+        out = output.get_formatted_output()
+        out_json = output.get_formatted_output("json")
+
+        try:
+            cpv = out[package]["metadata"]["cpv"]
+        except KeyError:
+            return {}
+
+        obj = self.store_package(cpv)
+
+        for res in out[package]["result"]:
+            self.store_version(obj, res["version"], " ".join(res["urls"]))
+
+        self.store_result(obj, out_json)
+
+        return out
+
+    def store_result(self, package, log):
+        # Remove previous logs
+        EuscanResult.objects.filter(package=package).delete()
+
+        obj = EuscanResult()
+        obj.package = package
+        obj.result = log
+        obj.datetime = timezone.now()
+        obj.save()
+
+    def store_package(self, cpv):
+        cat, pkg, ver, rev = portage.catpkgsplit(cpv)
+
+        obj, created = Package.objects.get_or_create(category=cat, name=pkg)
+
+        if created and not self.quiet:
+            sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
+
+        # Set all versions dead, then set found versions alive and
+        # delete old versions
+        Version.objects.filter(package=obj, packaged=False).update(alive=False)
+
+        return obj
+
+    def store_version(self, package, ver, url):
+        obj, created = Version.objects.get_or_create(
+            package=package, slot='', revision='r0', version=ver, overlay='',
+            defaults={"alive": True, "urls": url, "packaged": False}
+        )
+        if not created:
+            obj.alive = True
+            obj.urls = url
+            obj.packaged = False
+            obj.save()
+
+        # If it's not a new version, just update the object and continue
+        if not created:
+            return
+
+        if not self.quiet:
+            sys.stdout.write('+ [u] %s %s\n' % (obj, url))
+
+        VersionLog.objects.create(
+            package=package,
+            action=VersionLog.VERSION_ADDED,
+            slot='',
+            revision='r0',
+            version=ver,
+            overlay=''
+        )
+
+        package.n_versions += 1
+        package.save()
+
+
+@commit_on_success
+def purge_versions(quiet=False):
+    # For each dead versions
+    for version in Version.objects.filter(packaged=False, alive=False):
+        VersionLog.objects.create(
+            package=version.package,
+            action=VersionLog.VERSION_REMOVED,
+            slot=version.slot,
+            revision=version.revision,
+            version=version.version,
+            overlay=version.overlay
+        )
+
+        version.package.n_versions -= 1
+        version.package.save()
+
+        if not quiet:
+            sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
+    Version.objects.filter(packaged=False, alive=False).delete()
+
+
+def scan_upstream(packages=None, purge_versions=False, quiet=False,
+                  logger=None, stdout=None):
+
+    stdout = sys.stdout if stdout is None else stdout
+
+    scan_handler = ScanUpstream(quiet)
+
+    if not quiet:
+        stdout.write('Scanning upstream...\n')
+
+    if packages is None:
+        packages = Package.objects.all()
+
+    for pkg in packages:
+        if isinstance(pkg, Package):
+            scan_handler.scan('%s/%s' % (pkg.category, pkg.name))
+        else:
+            scan_handler.scan(pkg)
+
+    if purge_versions:
+        purge_versions(quiet)
+
+    if not quiet:
+        stdout.write('Done.\n')
diff --git a/euscanwww/djeuscan/processing/update_counters.py b/euscanwww/djeuscan/processing/update_counters.py
new file mode 100644
index 0000000..6d1c832
--- /dev/null
+++ b/euscanwww/djeuscan/processing/update_counters.py
@@ -0,0 +1,185 @@
+import sys
+
+from django.db.transaction import commit_on_success
+from django.utils import timezone
+
+from djeuscan.models import Package, Herd, Maintainer, Version
+from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
+from djeuscan import charts
+from djeuscan.processing import FakeLogger
+
+from distutils.version import StrictVersion, LooseVersion
+
+
+def _compare_versions(version1, version2):
+    try:
+        return cmp(StrictVersion(version1), StrictVersion(version2))
+    # in case of abnormal version number, fall back to LooseVersion
+    except ValueError:
+        return cmp(LooseVersion(version1), LooseVersion(version2))
+
+
+def _add_safe(storage, key):
+    if key not in storage:
+        storage[key] = 1
+    else:
+        storage[key] += 1
+
+
+def _add_last_ver(storage, version):
+    key = version['package_id']
+    if key not in storage:
+        storage[key] = version
+        return
+    if version['version'].startswith('9999'):
+        return
+    if _compare_versions(storage[key]['version'],
+                        version['version']) < 0:
+        storage[key] = version
+
+
+@commit_on_success
+def update_counters(stdout=None, fast=False, quiet=False, nolog=False,
+                    logger=None):
+    if logger is None:
+        logger = FakeLogger()  # TODO: write log messages
+
+    if stdout is None:
+        stdout = sys.stdout
+
+    now = timezone.now()
+
+    categories = {}
+    herds = {}
+    maintainers = {}
+
+    wlog = None
+
+    if not nolog:
+        wlog = WorldLog()
+        wlog.datetime = now
+
+        for cat in Package.objects.values('category').distinct():
+            clog = CategoryLog()
+            clog.datetime = now
+            clog.category = cat['category']
+            categories[clog.category] = clog
+
+        for herd in Herd.objects.all():
+            hlog = HerdLog()
+            hlog.datetime = now
+            hlog.herd = herd
+            herds[herd.id] = hlog
+
+        for maintainer in Maintainer.objects.all():
+            mlog = MaintainerLog()
+            mlog.datetime = now
+            mlog.maintainer = maintainer
+            maintainers[maintainer.id] = mlog
+
+    package_queryset = Package.objects.all()
+
+    n_versions = {}
+    n_packaged = {}
+    n_overlay = {}
+
+    last_versions_gentoo = {}
+    last_versions_overlay = {}
+    last_versions_upstream = {}
+
+    if not fast:
+        attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
+        for version in Version.objects.all().values(*attrs):
+            overlay, packaged = version['overlay'], version['packaged']
+            package_id = version['package_id']
+
+            _add_safe(n_versions, package_id)
+
+            if not packaged:
+                _add_last_ver(last_versions_upstream, version)
+                continue
+            if overlay == 'gentoo':
+                _add_safe(n_packaged, package_id)
+                _add_last_ver(last_versions_gentoo, version)
+            else:
+                _add_safe(n_overlay, package_id)
+                _add_last_ver(last_versions_overlay, version)
+
+    for package in package_queryset.select_related('herds', 'maintainers'):
+        if not fast:
+            package.n_versions = n_versions.get(package.id, 0)
+            package.n_packaged = n_packaged.get(package.id, 0)
+            package.n_overlay = n_overlay.get(package.id, 0)
+
+            default = {'id': None}
+            package.last_version_gentoo_id = last_versions_gentoo.get(
+                package.id, default
+            )['id']
+            package.last_version_overlay_id = last_versions_overlay.get(
+                package.id, default
+            )['id']
+            package.last_version_upstream_id = last_versions_upstream.get(
+                package.id, default
+            )['id']
+
+            package.save()
+
+        n_packages_gentoo = int(package.n_packaged == package.n_versions)
+        n_packages_overlay = int(package.n_overlay and package.n_packaged \
+                                 + package.n_overlay == package.n_versions)
+        n_packages_outdated = int(package.n_packaged + package.n_overlay \
+                                  < package.n_versions)
+
+        def update_row(storage, key):
+            storage[key].n_packages_gentoo += n_packages_gentoo
+            storage[key].n_packages_overlay += n_packages_overlay
+            storage[key].n_packages_outdated += n_packages_outdated
+
+            storage[key].n_versions_gentoo += package.n_packaged
+            storage[key].n_versions_overlay += package.n_overlay
+            storage[key].n_versions_upstream += package.n_versions - \
+                                                package.n_packaged - \
+                                                package.n_overlay
+
+        def update_log(storage, qs):
+            for row in qs:
+                update_row(storage, row['id'])
+
+        if not nolog:
+            update_log(herds, package.herds.all().values('id'))
+            update_log(maintainers, package.maintainers.all().values('id'))
+            update_row(categories, package.category)
+
+            wlog.n_packages_gentoo += n_packages_gentoo
+            wlog.n_packages_overlay += n_packages_overlay
+            wlog.n_packages_outdated += n_packages_outdated
+
+            wlog.n_versions_gentoo += package.n_packaged
+            wlog.n_versions_overlay += package.n_overlay
+            wlog.n_versions_upstream += package.n_versions - \
+                                        package.n_packaged - \
+                                        package.n_overlay
+
+    if nolog:
+        return
+
+    for clog in categories.values():
+        if not quiet:
+            stdout.write('+ [cl] %s\n' % clog)
+        charts.rrd_update('category-%s' % clog.category, now, clog)
+        clog.save()
+
+    for hlog in herds.values():
+        if not quiet:
+            stdout.write('+ [hl] %s\n' % hlog)
+        charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
+        hlog.save()
+
+    for mlog in maintainers.values():
+        if not quiet:
+            stdout.write('+ [ml] %s\n' % mlog)
+        charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
+        mlog.save()
+
+    charts.rrd_update('world', now, wlog)
+    wlog.save()
diff --git a/euscanwww/djeuscan/processing/update_portage_trees.py b/euscanwww/djeuscan/processing/update_portage_trees.py
new file mode 100644
index 0000000..cbf0a0b
--- /dev/null
+++ b/euscanwww/djeuscan/processing/update_portage_trees.py
@@ -0,0 +1,66 @@
+import subprocess
+from StringIO import StringIO
+
+from django.conf import settings
+
+from djeuscan.processing import FakeLogger
+
+
+def _launch_command(cmd):
+    """
+    Helper for launching shell commands inside tasks
+    """
+    fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+    output = StringIO(fp.communicate()[0])
+    return output.getvalue()
+
+
+def emerge_sync():
+    """
+    Launches an emerge --sync
+    """
+    cmd = ["emerge", "--sync", "--root", settings.PORTAGE_ROOT,
+           "--config-root", settings.PORTAGE_CONFIGROOT]
+    return _launch_command(cmd)
+
+
+def layman_sync():
+    """
+    Syncs Layman repos
+    """
+    from layman import Layman
+    l = Layman(config=settings.LAYMAN_CONFIG)
+    return l.sync(l.get_installed(), output_results=False)
+
+
+def emerge_regen():
+    """
+    Launches emerge --regen
+    """
+    cmd = [
+        "emerge", "--regen", "--jobs", settings.EMERGE_REGEN_JOBS, "--root",
+        settings.PORTAGE_ROOT, "--config-root", settings.PORTAGE_CONFIGROOT
+    ]
+    return _launch_command(cmd)
+
+
+def eix_update():
+    """
+    Launches eix-update
+    """
+    cmd = ["eix-update"]
+    return _launch_command(cmd)
+
+
+def update_portage_trees(logger=None):
+    logger = logger or FakeLogger()
+    logger.info("Running emerge --sync")
+    emerge_sync()
+    logger.info("Running layman --sync")
+    layman_sync()
+    logger.info("Running emerge --regen")
+    emerge_regen()
+    logger.info("Running eix-update")
+    eix_update()
+    logger.info("Done!")
diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index 290df65..2047ffe 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -1,23 +1,20 @@
-import subprocess
-from StringIO import StringIO
+"""
+Celery tasks for djeuscan
+"""
+
 from itertools import islice
 
 from celery.task import task, periodic_task
 from celery.task.schedules import crontab
 from celery.task.sets import TaskSet
 
-from django.conf import settings
-
-from euscan import output as euscan_output
-
 from djeuscan.models import Package, RefreshPackageQuery
-from djeuscan.management.commands.regen_rrds import regen_rrds
-from djeuscan.management.commands.update_counters import update_counters
-from djeuscan.management.commands.scan_metadata import ScanMetadata
-from djeuscan.management.commands.scan_portage import ScanPortage, \
-    purge_versions as scan_portage_purge
-from djeuscan.management.commands.scan_upstream import ScanUpstream, \
-    purge_versions as scan_upstream_purge
+from djeuscan.processing.regen_rrds import regen_rrds
+from djeuscan.processing.update_counters import update_counters
+from djeuscan.processing.scan_metadata import scan_metadata
+from djeuscan.processing.scan_portage import scan_portage
+from djeuscan.processing.scan_upstream import scan_upstream
+from djeuscan.processing.update_portage_trees import update_portage_trees
 
 
 class TaskFailedException(Exception):
@@ -27,16 +24,6 @@ class TaskFailedException(Exception):
     pass
 
 
-def _launch_command(cmd):
-    """
-    Helper for launching shell commands inside tasks
-    """
-    fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                          stderr=subprocess.PIPE)
-    output = StringIO(fp.communicate()[0])
-    return output.getvalue()
-
-
 def _chunks(it, n):
     """
     Chunk generator, takes an iterator and the desired size of the chunk
@@ -45,162 +32,231 @@ def _chunks(it, n):
         yield [first] + list(islice(it, n - 1))
 
 
-def _run_in_chunks(task, iterable, n=32):
+def _run_in_chunks(task, packages, kwargs=None, concurrently=8, n=32):
     """
-    Runs the given task with the given iterable of args in chunks of
-    n subtasks
+    Launches a TaskSet at a time with <concurrently> subtasks.
+    Each subtask has <n> packages to handle
     """
     output = []
-    for chunk in _chunks(iter(iterable), n):
-        job = TaskSet(tasks=[
-            task.subtask(args)
-            for args in chunk
-        ])
+
+    chunk_generator = _chunks(iter(packages), n)
+    done = False
+
+    while not done:
+        tasks = []
+        for _ in range(concurrently):
+            try:
+                chunk = chunk_generator.next()
+            except StopIteration:
+                done = True
+            else:
+                tasks.append(task.subtask((chunk, ), kwargs))
+        job = TaskSet(tasks=tasks)
         result = job.apply_async()
         # TODO: understand why this causes timeout
-        #output.extend(list(result.join(timeout=3600)))
+        output.extend(list(result.join(timeout=3600)))
     return output
 
 
 @task
 def regen_rrds_task():
+    """
+    Regenerate RRDs
+    """
     return regen_rrds()
 
 
 @task
-def update_counters_task():
-    return update_counters()
+def update_counters_task(fast=True):
+    """
+    Updates counters
+    """
+    return update_counters(fast=fast)
 
 
 @task
-def scan_metadata_task(query, obj=None):
-    logger = scan_metadata_task.get_logger()
-    logger.info("Starting metadata scanning for package %s ...", query)
+def _scan_metadata_task(packages):
+    """
+    Scans metadata for the given set of packages
+    """
+    logger = _scan_metadata_task.get_logger()
+    logger.info("Starting metadata scanning subtask for %d packages...",
+                len(packages))
 
-    scan_metadata = ScanMetadata()
-    result = scan_metadata.scan(query, obj)
+    result = scan_metadata(
+        packages=packages,
+        logger=logger,
+    )
     if not result:
-        raise TaskFailedException("Couldn't scan metadata")
+        raise TaskFailedException
     return result
 
 
 @task
 def scan_metadata_list_task(query):
-    return _run_in_chunks(scan_metadata_task, [(p, ) for p in query.split()])
+    """
+    Runs a parallel metadata scan for packages in the query list (space
+    separated string). Task used only from the web interface.
+    """
+    _run_in_chunks(_scan_metadata_task, [p for p in query.split()])
 
 
 @task
 def scan_metadata_all_task():
-    return _run_in_chunks(
-        scan_metadata_task,
-        [('%s/%s' % (pkg.category, pkg.name), pkg)
-         for pkg in Package.objects.all()]
+    """
+    Runs a parallel metadata scan for all packages
+    """
+    _run_in_chunks(_scan_metadata_task, Package.objects.all())
+
+
+@task
+def _scan_portage_task(packages, no_logs=False, purge_packages=False,
+                       purge_versions=False, prefetch=False):
+    """
+    Scans portage for the given set of packages
+    """
+    logger = _scan_portage_task.get_logger()
+    logger.info("Starting portage scanning subtask for %d packages...",
+                len(packages))
+
+    result = scan_portage(
+        packages=packages,
+        no_logs=no_logs,
+        purge_packages=purge_packages,
+        purge_versions=purge_versions,
+        prefetch=prefetch,
+        logger=logger,
     )
-
-
-@task
-def scan_portage_list_task(query, purge=False):
-    scan_portage = ScanPortage()
-    logger = scan_portage_list_task.get_logger()
-
-    for pkg in query.split():
-        logger.info("Starting Portage package scanning: %s ...", pkg)
-
-        scan_portage.scan(pkg)
-
-    if purge:
-        logger.info("Purging")
-        scan_portage_purge()
-
-
-@task
-def scan_portage_all_task(purge=False):
-    logger = scan_portage_all_task.get_logger()
-    logger.info("Starting Portage scanning...")
-
-    scan_portage = ScanPortage()
-    scan_portage.scan()
-
-    if purge:
-        logger.info("Purging")
-        scan_portage_purge()
-
-
-@task
-def scan_upstream_task(query):
-    logger = scan_upstream_task.get_logger()
-    logger.info("Starting upstream scanning for package %s ...", query)
-
-    euscan_output.clean()
-    scan_upstream = ScanUpstream()
-    result = scan_upstream.scan(query)
-    euscan_output.clean()
-    if not result or result == {}:
-        raise TaskFailedException("Couldn't scan upstream")
+    if not result:
+        raise TaskFailedException
     return result
 
 
 @task
-def scan_upstream_list_task(query):
-    return _run_in_chunks(scan_upstream_task, [(p, ) for p in query.split()])
+def scan_portage_list_task(query, no_logs=False, purge_packages=False,
+                           purge_versions=False, prefetch=False):
+    """
+    Runs a parallel portage scan for packages in the query list (space
+    separated string). Task used only from the web interface.
+    """
+    kwargs = {"no_logs": no_logs, "purge_packages": purge_packages,
+              "purge_versions": purge_versions, "prefetch": prefetch}
+    _run_in_chunks(_scan_portage_task, [p for p in query.split()], kwargs)
 
 
 @task
-def scan_upstream_all_task(purge=False):
-    output = _run_in_chunks(
-        scan_upstream_task,
-        [('%s/%s' % (pkg.category, pkg.name), )
-         for pkg in Package.objects.all()],
-        n=16
+def scan_portage_all_task(no_logs=False, purge_packages=False,
+                          purge_versions=False, prefetch=False):
+    """
+    Runs a parallel portage scan for all packages
+    """
+    kwargs = {"no_logs": no_logs, "purge_packages": purge_packages,
+              "purge_versions": purge_versions, "prefetch": prefetch}
+    _run_in_chunks(_scan_metadata_task, Package.objects.all(), kwargs)
+
+
+@task
+def _scan_upstream_task(packages, purge_versions=False):
+    """
+    Scans upstream for the given set of packages
+    """
+    logger = _scan_upstream_task.get_logger()
+
+    logger.info("Starting upstream scanning subtask for %d packages...",
+                len(packages))
+
+    result = scan_upstream(
+        packages=packages,
+        purge_versions=purge_versions,
+        logger=logger,
     )
-
-    if purge:
-        output += [scan_upstream_purge()]
-
-    return output
+    if not result:
+        raise TaskFailedException
+    return result
 
 
 @task
-def emerge_sync():
-    cmd = ["emerge", "--sync", "--root", settings.PORTAGE_ROOT,
-           "--config-root", settings.PORTAGE_CONFIGROOT]
-    return _launch_command(cmd)
+def scan_upstream_list_task(query, purge_versions=False):
+    """
+    Runs a parallel upstream scan for packages in the query list (space
+    separated string). Task used only from the web interface.
+    """
+
+    kwargs = {"purge_versions": purge_versions}
+    _run_in_chunks(_scan_upstream_task, [p for p in query.split()], kwargs)
 
 
 @task
-def layman_sync():
-    from layman import Layman
-    l = Layman(config=settings.LAYMAN_CONFIG)
-    return l.sync(l.get_installed(), output_results=False)
+def scan_upstream_all_task(purge_versions=False):
+    """
+    Runs a parallel portage scan for all packages
+    """
+    kwargs = {"purge_versions": purge_versions}
+    _run_in_chunks(_scan_upstream_task, Package.objects.all(), kwargs)
 
 
 @task
-def emerge_regen():
-    cmd = [
-        "emerge", "--regen", "--jobs", settings.EMERGE_REGEN_JOBS, "--root",
-        settings.PORTAGE_ROOT, "--config-root", settings.PORTAGE_CONFIGROOT
-    ]
-    return _launch_command(cmd)
+def update_portage_trees_task():
+    """
+    Update portage tree
+    """
+    logger = update_portage_trees_task.get_logger()
+    update_portage_trees(logger=logger)
 
 
 @task
-def eix_update():
-    cmd = ["eix-update"]
-    return _launch_command(cmd)
+def update_task(update_portage_trees=True, scan_portage=True,
+                scan_metadata=True, scan_upstream=True, update_counter=True):
+    """
+    Update the whole euscan system
+    """
+    if update_portage_trees:
+        update_portage_trees_task()
+    if scan_portage:
+        scan_portage_all_task(prefetch=True, purge_packages=True,
+                              purge_versions=True)
+
+    # metadata and upstream scan can run concurrently, launch them
+    # asynchronously and wait for them to finish
+    metadata_job = None
+    if scan_metadata:
+        metadata_job = scan_metadata_all_task().delay()
+
+    upstream_job = None
+    if scan_upstream:
+        upstream_job = scan_upstream_all_task().delay()
+
+    if metadata_job:
+        metadata_job.wait()
+    if upstream_job:
+        upstream_job.wait()
+
+    update_counters(fast=False)
 
 
 @periodic_task(run_every=crontab(minute="*/1"))
-def refresh_package_consume():
+def consume_refresh_package_request():
+    """
+    Satisfies user requests for package refreshing, runs every minute
+    """
     try:
         obj = RefreshPackageQuery.objects.latest()
     except RefreshPackageQuery.DoesNotExist:
         return {}
     else:
-        result = scan_upstream_task(obj.query)
+        result = _scan_upstream_task([obj.query])
         obj.delete()
         return result
 
 
+@periodic_task(run_every=crontab(hour=03, minute=00, day_of_week=1))
+def update_periodic_task():
+    """
+    Runs a whole update once a week
+    """
+    update_task()
+
+
 admin_tasks = [
     regen_rrds_task,
     update_counters_task,
@@ -210,8 +266,6 @@ admin_tasks = [
     scan_portage_list_task,
     scan_upstream_all_task,
     scan_upstream_list_task,
-    emerge_sync,
-    layman_sync,
-    emerge_regen,
-    eix_update,
+    update_portage_trees,
+    update_task
 ]

From 779ccdf5d32024f626bf42c5c4b74d5fcb4c14cc Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Fri, 22 Jun 2012 11:31:15 +0200
Subject: [PATCH 26/37] euscanwww: use logging in djeuscan.processing

Signed-off-by: volpino <fox91@anche.no>
---
 .../management/commands/regen_rrds.py         |  8 ++-
 .../management/commands/scan_metadata.py      | 13 ++--
 .../management/commands/scan_portage.py       | 15 ++---
 .../management/commands/scan_upstream.py      | 14 ++---
 .../management/commands/update_counters.py    | 13 ++--
 euscanwww/djeuscan/processing/__init__.py     | 28 +++++++++
 .../djeuscan/processing/scan_metadata.py      | 37 ++++++-----
 euscanwww/djeuscan/processing/scan_portage.py | 62 +++++++------------
 .../djeuscan/processing/scan_upstream.py      | 37 +++++------
 .../djeuscan/processing/update_counters.py    | 20 ++----
 euscanwww/euscanwww/settings.py               | 15 +++++
 11 files changed, 142 insertions(+), 120 deletions(-)

diff --git a/euscanwww/djeuscan/management/commands/regen_rrds.py b/euscanwww/djeuscan/management/commands/regen_rrds.py
index f6b2ba0..a53f44d 100644
--- a/euscanwww/djeuscan/management/commands/regen_rrds.py
+++ b/euscanwww/djeuscan/management/commands/regen_rrds.py
@@ -1,10 +1,16 @@
+import logging
 from django.core.management.base import BaseCommand
+
+from djeuscan.processing import set_verbosity_level
 from djeuscan.processing.regen_rrds import regen_rrds
 
+logger = logging.getLogger(__name__)
+
 
 class Command(BaseCommand):
     _overlays = {}
     help = 'Regenerate rrd database'
 
     def handle(self, *args, **options):
-        regen_rrds()
+        set_verbosity_level(logger, options.get("verbosity", 1))
+        regen_rrds(logger=logger)
diff --git a/euscanwww/djeuscan/management/commands/scan_metadata.py b/euscanwww/djeuscan/management/commands/scan_metadata.py
index 5a19315..5decb70 100644
--- a/euscanwww/djeuscan/management/commands/scan_metadata.py
+++ b/euscanwww/djeuscan/management/commands/scan_metadata.py
@@ -1,10 +1,14 @@
 import sys
+import logging
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
 
+from djeuscan.processing import set_verbosity_level
 from djeuscan.processing.scan_metadata import scan_metadata
 
+logger = logging.getLogger(__name__)
+
 
 class Command(BaseCommand):
     _overlays = {}
@@ -15,16 +19,13 @@ class Command(BaseCommand):
             dest='all',
             default=False,
             help='Scan all packages'),
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
         )
     args = '<package package ...>'
     help = 'Scans metadata and fills database'
 
     def handle(self, *args, **options):
+        set_verbosity_level(logger, options.get("verbosity", 1))
+
         if options['all']:
             packages = None
 
@@ -33,4 +34,4 @@ class Command(BaseCommand):
         else:
             packages = [pkg[:-1] for pkg in sys.stdin.readlines()]
 
-        scan_metadata(packages=packages, quiet=options["quiet"])
+        scan_metadata(packages=packages, logger=logger)
diff --git a/euscanwww/djeuscan/management/commands/scan_portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py
index 2bfede8..c0dd6fe 100644
--- a/euscanwww/djeuscan/management/commands/scan_portage.py
+++ b/euscanwww/djeuscan/management/commands/scan_portage.py
@@ -1,9 +1,14 @@
 import sys
+import logging
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
+
+from djeuscan.processing import set_verbosity_level
 from djeuscan.processing.scan_portage import scan_portage
 
+logger = logging.getLogger(__name__)
+
 
 class Command(BaseCommand):
     _overlays = {}
@@ -35,16 +40,13 @@ class Command(BaseCommand):
             default=False,
             help=('Prefetch all versions and packages from DB to '
                   'speedup full scan process.')),
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
         )
     args = '[package package ...]'
     help = 'Scans portage tree and fills database'
 
     def handle(self, *args, **options):
+        set_verbosity_level(logger, options.get("verbosity", 1))
+
         if options['all']:
             packages = None
 
@@ -59,6 +61,5 @@ class Command(BaseCommand):
             purge_packages=options["purge-packages"],
             purge_versions=options["purge-versions"],
             prefetch=options["prefetch"],
-            quiet=options["quiet"],
-            stdout=self.stdout,
+            logger=logger,
         )
diff --git a/euscanwww/djeuscan/management/commands/scan_upstream.py b/euscanwww/djeuscan/management/commands/scan_upstream.py
index 043fb83..f185c38 100644
--- a/euscanwww/djeuscan/management/commands/scan_upstream.py
+++ b/euscanwww/djeuscan/management/commands/scan_upstream.py
@@ -1,10 +1,14 @@
+import logging
 import sys
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
 
+from djeuscan.processing import set_verbosity_level
 from djeuscan.processing.scan_upstream import scan_upstream
 
+logger = logging.getLogger(__name__)
+
 
 class Command(BaseCommand):
     _overlays = {}
@@ -20,16 +24,13 @@ class Command(BaseCommand):
             dest='purge-versions',
             default=False,
             help='Purge old versions'),
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
         )
     args = '<package package ...>'
     help = 'Scans metadata and fills database'
 
     def handle(self, *args, **options):
+        set_verbosity_level(logger, options.get("verbosity", 1))
+
         if options['all']:
             packages = None
 
@@ -41,6 +42,5 @@ class Command(BaseCommand):
         scan_upstream(
             packages=packages,
             purge_versions=options["purge-versions"],
-            quiet=options["quiet"],
-            stdout=self.stdout,
+            logger=logger,
         )
diff --git a/euscanwww/djeuscan/management/commands/update_counters.py b/euscanwww/djeuscan/management/commands/update_counters.py
index 7dc407e..100d5a7 100644
--- a/euscanwww/djeuscan/management/commands/update_counters.py
+++ b/euscanwww/djeuscan/management/commands/update_counters.py
@@ -1,20 +1,19 @@
+import logging
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
 
+from djeuscan.processing import set_verbosity_level
 from djeuscan.processing.update_counters import update_counters
 
+logger = logging.getLogger(__name__)
+
 
 class Command(BaseCommand):
     _overlays = {}
     help = 'Update counters'
 
     option_list = BaseCommand.option_list + (
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
         make_option('--fast',
             action='store_true',
             dest='fast',
@@ -28,9 +27,9 @@ class Command(BaseCommand):
     )
 
     def handle(self, *args, **options):
+        set_verbosity_level(logger, options.get("verbosity", 1))
         update_counters(
-            stdout=self.stdout,
             fast=options["fast"],
-            quiet=options["quiet"],
             nolog=options["nolog"],
+            logger=logger,
         )
diff --git a/euscanwww/djeuscan/processing/__init__.py b/euscanwww/djeuscan/processing/__init__.py
index c4ea508..4a203b7 100644
--- a/euscanwww/djeuscan/processing/__init__.py
+++ b/euscanwww/djeuscan/processing/__init__.py
@@ -1,3 +1,31 @@
+import logging
+
+
 class FakeLogger(object):
     def __getattr__(self, key):
         return lambda *x, **y: None
+
+
+def set_verbosity_level(logger, verbosity):
+    try:
+        verbosity = int(verbosity)
+    except (ValueError, TypeError):
+        return logger
+
+    levels = {
+        0: logging.DEBUG,
+        1: logging.INFO,
+        2: logging.WARNING,
+        3: logging.ERROR,
+        4: logging.CRITICAL
+    }
+
+    if verbosity < 0:
+        verbosity = 0
+
+    if verbosity > 4:
+        verbosity = 4
+
+    logger.setLevel(levels[verbosity])
+
+    return logger
diff --git a/euscanwww/djeuscan/processing/scan_metadata.py b/euscanwww/djeuscan/processing/scan_metadata.py
index 15ab108..4f06d05 100644
--- a/euscanwww/djeuscan/processing/scan_metadata.py
+++ b/euscanwww/djeuscan/processing/scan_metadata.py
@@ -1,4 +1,3 @@
-import sys
 from gentoolkit.query import Query
 from gentoolkit.errors import GentoolkitFatalError
 
@@ -7,12 +6,13 @@ from django.core.management.color import color_style
 from django.core.exceptions import ValidationError
 
 from djeuscan.models import Package, Herd, Maintainer
+from djeuscan.processing import FakeLogger
 
 
 class ScanMetadata(object):
-    def __init__(self, quiet=False):
-        self.quiet = quiet
+    def __init__(self, logger=None):
         self.style = color_style()
+        self.logger = logger or FakeLogger()
 
     @commit_on_success
     def scan(self, query=None, obj=None):
@@ -22,8 +22,8 @@ class ScanMetadata(object):
         )
 
         if not matches:
-            sys.stderr.write(
-                self.style.ERROR("Unknown package '%s'\n" % query)
+            self.logger.error(
+                self.style.ERROR("Unknown package '%s'" % query)
             )
             return
 
@@ -43,14 +43,14 @@ class ScanMetadata(object):
             obj.homepage = pkg.environment("HOMEPAGE")
             obj.description = pkg.environment("DESCRIPTION")
         except GentoolkitFatalError, err:
-            sys.stderr.write(
+            self.logger.error(
                 self.style.ERROR(
-                    "Gentoolkit fatal error: '%s'\n" % str(err)
+                    "Gentoolkit fatal error: '%s'" % str(err)
                 )
             )
 
-        if created and not self.quiet:
-            sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
+        if created:
+            self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name))
 
         if pkg.metadata:
             herds = dict(
@@ -92,8 +92,8 @@ class ScanMetadata(object):
                     )
                     obj.maintainers.add(maintainer)
                 except ValidationError:
-                    sys.stderr.write(
-                        self.style.ERROR("Bad maintainer: '%s' '%s'\n" % \
+                    self.logger.error(
+                        self.style.ERROR("Bad maintainer: '%s' '%s'" % \
                                          (maintainer.name, maintainer.email))
                     )
 
@@ -109,8 +109,8 @@ class ScanMetadata(object):
             defaults={"email": email}
         )
 
-        if created and not self.quiet:
-            sys.stdout.write('+ [h] %s <%s>\n' % (name, email))
+        if created:
+            self.logger.info('+ [h] %s <%s>' % (name, email))
 
         herd.email = email
         herd.save()
@@ -129,15 +129,14 @@ class ScanMetadata(object):
         )
 
         if created:
-            if not self.quiet:
-                sys.stdout.write(
-                    '+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
-                )
+            self.logger.info(
+                '+ [m] %s <%s>' % (name.encode('utf-8'), email)
+            )
         return maintainer
 
 
-def scan_metadata(packages=None, quiet=False, logger=None):
-    scan_handler = ScanMetadata(quiet=quiet)
+def scan_metadata(packages=None, logger=None):
+    scan_handler = ScanMetadata(logger=logger)
     if packages is None:
         packages = Package.objects.all()
 
diff --git a/euscanwww/djeuscan/processing/scan_portage.py b/euscanwww/djeuscan/processing/scan_portage.py
index 43e91f4..fdebc57 100644
--- a/euscanwww/djeuscan/processing/scan_portage.py
+++ b/euscanwww/djeuscan/processing/scan_portage.py
@@ -1,24 +1,23 @@
 import subprocess
 import portage
-import sys
 import os
 import re
 
 from django.db.transaction import commit_on_success
 from django.core.management.color import color_style
 
+from djeuscan.processing import FakeLogger
 from djeuscan.models import Package, Version, VersionLog
 
 
 class ScanPortage(object):
-    def __init__(self, stdout=None, no_log=False, purge_packages=False,
-                 purge_versions=False, kill_versions=False, quiet=False):
-        self.stdout = sys.stdout if stdout is None else stdout
+    def __init__(self, logger=None, no_log=False, purge_packages=False,
+                 purge_versions=False, kill_versions=False):
+        self.logger = logger or FakeLogger()
         self.no_log = no_log
         self.purge_packages = purge_packages
         self.purge_versions = purge_versions
         self.kill_versions = kill_versions
-        self.quiet = quiet
 
         self.style = color_style()
         self._cache = {'packages': {}, 'versions': {}}
@@ -93,12 +92,9 @@ class ScanPortage(object):
             cmd.extend(['--exact', query])
 
         if self.kill_versions:
-            if not self.quiet:
-                self.stdout.write('Killing existing versions...')
-                self.stdout.flush()
+            self.logger.info('Killing existing versions...')
             Version.objects.filter(packaged=True).update(alive=False)
-            if not self.quiet:
-                self.stdout.write('done\n')
+            self.logger.info('done')
 
         output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
             communicate()[0]
@@ -108,17 +104,16 @@ class ScanPortage(object):
             if not query:
                 return
             if self.purge_packages:
-                if not self.quiet:
-                    sys.stdout.write('- [p] %s\n' % (query))
+                self.logger.info('- [p] %s' % (query))
                 if '/' in query:
                     cat, pkg = portage.catsplit(query)
                     Package.objects.filter(category=cat, name=pkg).delete()
                 else:
                     Package.objects.filter(name=query).delete()
             else:
-                sys.stderr.write(
+                self.logger.error(
                     self.style.ERROR(
-                        "Unknown package '%s'\n" % query
+                        "Unknown package '%s'" % query
                     )
                 )
             return
@@ -156,8 +151,7 @@ class ScanPortage(object):
             for package in Package.objects.all():
                 cp = "%s/%s" % (package.category, package.name)
                 if cp not in packages:
-                    if not self.quiet:
-                        sys.stdout.write('- [p] %s\n' % (package))
+                    self.logger.info('- [p] %s' % (package))
                     package.delete()
 
     def store_package(self, cat, pkg):
@@ -172,8 +166,7 @@ class ScanPortage(object):
             self.cache_store_package(obj)
 
         if created:
-            if not self.quiet:
-                sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
+            self.logger.info('+ [p] %s/%s' % (cat, pkg))
 
         # Set all versions dead, then set found versions alive and
         # delete old versions
@@ -219,8 +212,7 @@ class ScanPortage(object):
         if not created:
             return
 
-        if not self.quiet:
-            sys.stdout.write('+ [v] %s \n' % (obj))
+        self.logger.info('+ [v] %s' % (obj))
 
         if overlay == 'gentoo':
             package.n_packaged += 1
@@ -243,7 +235,9 @@ class ScanPortage(object):
 
 
 @commit_on_success
-def purge_versions(quiet=False, nolog=False):
+def purge_versions(logger=None, nolog=False):
+    logger = logger or FakeLogger()
+
     # For each dead versions
     for version in Version.objects.filter(packaged=True, alive=False):
         if version.overlay == 'gentoo':
@@ -253,8 +247,7 @@ def purge_versions(quiet=False, nolog=False):
         version.package.n_versions -= 1
         version.package.save()
 
-        if not quiet:
-            sys.stdout.write('- [v] %s\n' % (version))
+        logger.info('- [v] %s' % (version))
 
         if nolog:
             continue
@@ -272,37 +265,31 @@ def purge_versions(quiet=False, nolog=False):
 
 
 def scan_portage(packages=None, no_log=False, purge_packages=False,
-                 purge_versions=False, prefetch=False, logger=None,
-                 quiet=False, stdout=None):
-    stdout = sys.stdout if stdout is None else stdout
+                 purge_versions=False, prefetch=False, logger=None):
 
+    logger = logger or FakeLogger()
     kill_versions = False
     if packages is None:
         prefetch = True
         kill_versions = True
 
     scan_handler = ScanPortage(
-        stdout=stdout,
+        logger=logger,
         no_log=no_log,
         purge_packages=purge_packages,
         purge_versions=purge_versions,
         kill_versions=kill_versions,
-        quiet=quiet,
     )
 
-    if not quiet:
-        stdout.write('Scanning portage tree...\n')
+    logger.info('Scanning portage tree...')
 
     if prefetch:
-        if quiet:
-            stdout.write('Prefetching objects...')
-            stdout.flush()
+        logger.info('Prefetching objects...')
         for package in Package.objects.all():
             scan_handler.cache_store_package(package)
         for version in Version.objects.select_related('package').all():
             scan_handler.cache_store_version(version)
-        if quiet:
-            stdout.write('done\n')
+        logger.info('done')
 
     if packages is None:
         scan_handler.scan()
@@ -314,7 +301,6 @@ def scan_portage(packages=None, no_log=False, purge_packages=False,
                 scan_handler.scan(pkg)
 
     if purge_versions:
-        purge_versions(quiet, no_log)
+        purge_versions(logger=logger, no_log=no_log)
 
-    if not quiet:
-        stdout.write('Done.\n')
+    logger.info('Done.')
diff --git a/euscanwww/djeuscan/processing/scan_upstream.py b/euscanwww/djeuscan/processing/scan_upstream.py
index 9eee48b..2791ce5 100644
--- a/euscanwww/djeuscan/processing/scan_upstream.py
+++ b/euscanwww/djeuscan/processing/scan_upstream.py
@@ -1,5 +1,4 @@
 import portage
-import sys
 
 from django.utils import timezone
 from django.db.transaction import commit_on_success
@@ -7,12 +6,13 @@ from django.db.transaction import commit_on_success
 from euscan import CONFIG, output
 from euscan.scan import scan_upstream as euscan_scan_upstream
 
+from djeuscan.processing import FakeLogger
 from djeuscan.models import Package, Version, EuscanResult, VersionLog
 
 
 class ScanUpstream(object):
-    def __init__(self, quiet=False):
-        self.quiet = quiet
+    def __init__(self, logger=None):
+        self.logger = logger or FakeLogger()
 
     def scan(self, package):
         CONFIG["format"] = "dict"
@@ -52,8 +52,8 @@ class ScanUpstream(object):
 
         obj, created = Package.objects.get_or_create(category=cat, name=pkg)
 
-        if created and not self.quiet:
-            sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
+        if created:
+            self.logger.info('+ [p] %s/%s' % (cat, pkg))
 
         # Set all versions dead, then set found versions alive and
         # delete old versions
@@ -76,8 +76,7 @@ class ScanUpstream(object):
         if not created:
             return
 
-        if not self.quiet:
-            sys.stdout.write('+ [u] %s %s\n' % (obj, url))
+        self.logger.info('+ [u] %s %s' % (obj, url))
 
         VersionLog.objects.create(
             package=package,
@@ -93,7 +92,9 @@ class ScanUpstream(object):
 
 
 @commit_on_success
-def purge_versions(quiet=False):
+def purge_versions(logger=None):
+    logger = logger or FakeLogger()
+
     # For each dead versions
     for version in Version.objects.filter(packaged=False, alive=False):
         VersionLog.objects.create(
@@ -108,20 +109,17 @@ def purge_versions(quiet=False):
         version.package.n_versions -= 1
         version.package.save()
 
-        if not quiet:
-            sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
+        logger.info('- [u] %s %s' % (version, version.urls))
     Version.objects.filter(packaged=False, alive=False).delete()
 
 
-def scan_upstream(packages=None, purge_versions=False, quiet=False,
-                  logger=None, stdout=None):
+def scan_upstream(packages=None, purge_versions=False,
+                  logger=None):
+    logger = logger or FakeLogger()
 
-    stdout = sys.stdout if stdout is None else stdout
+    scan_handler = ScanUpstream(logger=logger)
 
-    scan_handler = ScanUpstream(quiet)
-
-    if not quiet:
-        stdout.write('Scanning upstream...\n')
+    logger.info('Scanning upstream...')
 
     if packages is None:
         packages = Package.objects.all()
@@ -133,7 +131,6 @@ def scan_upstream(packages=None, purge_versions=False, quiet=False,
             scan_handler.scan(pkg)
 
     if purge_versions:
-        purge_versions(quiet)
+        purge_versions(logger=logger)
 
-    if not quiet:
-        stdout.write('Done.\n')
+    logger.info('Done.')
diff --git a/euscanwww/djeuscan/processing/update_counters.py b/euscanwww/djeuscan/processing/update_counters.py
index 6d1c832..54cafed 100644
--- a/euscanwww/djeuscan/processing/update_counters.py
+++ b/euscanwww/djeuscan/processing/update_counters.py
@@ -1,5 +1,3 @@
-import sys
-
 from django.db.transaction import commit_on_success
 from django.utils import timezone
 
@@ -39,13 +37,8 @@ def _add_last_ver(storage, version):
 
 
 @commit_on_success
-def update_counters(stdout=None, fast=False, quiet=False, nolog=False,
-                    logger=None):
-    if logger is None:
-        logger = FakeLogger()  # TODO: write log messages
-
-    if stdout is None:
-        stdout = sys.stdout
+def update_counters(fast=False, nolog=False, logger=None):
+    logger = logger or FakeLogger()
 
     now = timezone.now()
 
@@ -164,20 +157,17 @@ def update_counters(stdout=None, fast=False, quiet=False, nolog=False,
         return
 
     for clog in categories.values():
-        if not quiet:
-            stdout.write('+ [cl] %s\n' % clog)
+        logger.info('+ [cl] %s\n' % clog)
         charts.rrd_update('category-%s' % clog.category, now, clog)
         clog.save()
 
     for hlog in herds.values():
-        if not quiet:
-            stdout.write('+ [hl] %s\n' % hlog)
+        logger.info('+ [hl] %s\n' % hlog)
         charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
         hlog.save()
 
     for mlog in maintainers.values():
-        if not quiet:
-            stdout.write('+ [ml] %s\n' % mlog)
+        logger.info('+ [ml] %s\n' % mlog)
         charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
         mlog.save()
 
diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py
index 2e7628c..116fcd9 100644
--- a/euscanwww/euscanwww/settings.py
+++ b/euscanwww/euscanwww/settings.py
@@ -181,12 +181,22 @@ INSTALLED_APPS = (
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
+    'formatters': {
+        'simple': {
+            'format': '%(levelname)s %(asctime)s %(message)s'
+        },
+    },
     'filters': {
         'require_debug_false': {
             '()': 'django.utils.log.RequireDebugFalse'
         }
     },
     'handlers': {
+        'console': {
+            'level': 'INFO',
+            'class': 'logging.StreamHandler',
+            'formatter': 'simple'
+        },
         'mail_admins': {
             'level': 'ERROR',
             'filters': ['require_debug_false'],
@@ -199,6 +209,11 @@ LOGGING = {
             'level': 'ERROR',
             'propagate': True,
         },
+        'djeuscan': {
+            'handlers': ['console'],
+            'level': 'INFO',
+            'propagate': True
+        }
     }
 }
 

From f9c9a4bfd8483bcf2b827f55bc5f1855f0f48af4 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Fri, 22 Jun 2012 11:40:40 +0200
Subject: [PATCH 27/37] euscanwww: added scan_package task

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/tasks.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index 2047ffe..8dfbeaa 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -234,6 +234,13 @@ def update_task(update_portage_trees=True, scan_portage=True,
     update_counters(fast=False)
 
 
+@task
+def scan_package_task(package):
+    _scan_portage_task([package], purge_packages=True, purge_versions=True)
+    _scan_metadata_task([package])
+    _scan_upstream_task([package])
+
+
 @periodic_task(run_every=crontab(minute="*/1"))
 def consume_refresh_package_request():
     """
@@ -244,7 +251,7 @@ def consume_refresh_package_request():
     except RefreshPackageQuery.DoesNotExist:
         return {}
     else:
-        result = _scan_upstream_task([obj.query])
+        result = scan_package_task(obj.query)
         obj.delete()
         return result
 
@@ -267,5 +274,6 @@ admin_tasks = [
     scan_upstream_all_task,
     scan_upstream_list_task,
     update_portage_trees,
-    update_task
+    update_task,
+    scan_package_task,
 ]

From d7e0a4ca16264249bfb26a419d3122d74d87c93b Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Tue, 26 Jun 2012 17:16:02 +0200
Subject: [PATCH 28/37] euscanwww: Tweaked processing.scan_portage to use eix
 --xml

Signed-off-by: volpino <fox91@anche.no>
---
 ..._type__add_field_version_handler__add_f.py | 197 ++++++++++++++++++
 euscanwww/djeuscan/models.py                  |   4 +
 euscanwww/djeuscan/processing/scan_portage.py |  73 ++++---
 pym/euscan/helpers.py                         |   2 +-
 4 files changed, 237 insertions(+), 39 deletions(-)
 create mode 100644 euscanwww/djeuscan/migrations/0005_auto__add_field_version_version_type__add_field_version_handler__add_f.py

diff --git a/euscanwww/djeuscan/migrations/0005_auto__add_field_version_version_type__add_field_version_handler__add_f.py b/euscanwww/djeuscan/migrations/0005_auto__add_field_version_version_type__add_field_version_handler__add_f.py
new file mode 100644
index 0000000..64eda3c
--- /dev/null
+++ b/euscanwww/djeuscan/migrations/0005_auto__add_field_version_version_type__add_field_version_handler__add_f.py
@@ -0,0 +1,197 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'Version.version_type'
+        db.add_column('djeuscan_version', 'version_type',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True),
+                      keep_default=False)
+
+        # Adding field 'Version.handler'
+        db.add_column('djeuscan_version', 'handler',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True),
+                      keep_default=False)
+
+        # Adding field 'Version.confidence'
+        db.add_column('djeuscan_version', 'confidence',
+                      self.gf('django.db.models.fields.IntegerField')(default=0),
+                      keep_default=False)
+
+    def backwards(self, orm):
+        # Deleting field 'Version.version_type'
+        db.delete_column('djeuscan_version', 'version_type')
+
+        # Deleting field 'Version.handler'
+        db.delete_column('djeuscan_version', 'handler')
+
+        # Deleting field 'Version.confidence'
+        db.delete_column('djeuscan_version', 'confidence')
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'djeuscan.categoryassociation': {
+            'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.categorylog': {
+            'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.euscanresult': {
+            'Meta': {'object_name': 'EuscanResult'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'djeuscan.herd': {
+            'Meta': {'object_name': 'Herd'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
+            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'djeuscan.herdassociation': {
+            'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.herdlog': {
+            'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.log': {
+            'Meta': {'object_name': 'Log'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'djeuscan.maintainer': {
+            'Meta': {'object_name': 'Maintainer'},
+            'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.maintainerassociation': {
+            'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.maintainerlog': {
+            'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
+        },
+        'djeuscan.package': {
+            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
+            'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.packageassociation': {
+            'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.refreshpackagequery': {
+            'Meta': {'object_name': 'RefreshPackageQuery'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
+        },
+        'djeuscan.version': {
+            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
+            'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'version_type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
+        },
+        'djeuscan.versionlog': {
+            'Meta': {'object_name': 'VersionLog'},
+            'action': ('django.db.models.fields.IntegerField', [], {}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.worldlog': {
+            'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        }
+    }
+
+    complete_apps = ['djeuscan']
\ No newline at end of file
diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py
index 936b2fc..5bca337 100644
--- a/euscanwww/djeuscan/models.py
+++ b/euscanwww/djeuscan/models.py
@@ -126,6 +126,10 @@ class Version(models.Model):
     urls = models.TextField(blank=True)
     alive = models.BooleanField(default=True, db_index=True)
 
+    version_type = models.CharField(max_length=128, blank=True)
+    handler = models.CharField(max_length=128, blank=True)
+    confidence = models.IntegerField(default=0)
+
     class Meta:
         unique_together = ['package', 'slot', 'revision', 'version', 'overlay']
 
diff --git a/euscanwww/djeuscan/processing/scan_portage.py b/euscanwww/djeuscan/processing/scan_portage.py
index fdebc57..34456ea 100644
--- a/euscanwww/djeuscan/processing/scan_portage.py
+++ b/euscanwww/djeuscan/processing/scan_portage.py
@@ -2,10 +2,13 @@ import subprocess
 import portage
 import os
 import re
+from xml.dom.minidom import parseString
 
 from django.db.transaction import commit_on_success
 from django.core.management.color import color_style
 
+from euscan.helpers import get_version_type
+
 from djeuscan.processing import FakeLogger
 from djeuscan.models import Package, Version, VersionLog
 
@@ -83,11 +86,7 @@ class ScanPortage(object):
 
     @commit_on_success
     def scan(self, query=None):
-        env = os.environ
-        env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
-
-        cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages',
-               '-x']
+        cmd = ['eix', '--xml', '--pure-packages', '-x']
         if query:
             cmd.extend(['--exact', query])
 
@@ -96,7 +95,7 @@ class ScanPortage(object):
             Version.objects.filter(packaged=True).update(alive=False)
             self.logger.info('done')
 
-        output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
+        output = subprocess.Popen(cmd, stdout=subprocess.PIPE).\
             communicate()[0]
         output = output.strip().strip('\n')
 
@@ -118,50 +117,42 @@ class ScanPortage(object):
                 )
             return
 
-        output = output.split('\n')
-        packages = {}
+        dom = parseString(output)
 
-        line_re = re.compile(
-            r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$'
-        )
+        for category_tag in dom.getElementsByTagName("category"):
+            for package_tag in category_tag.getElementsByTagName("package"):
+                cat = category_tag.getAttribute("name")
+                pkg = package_tag.getAttribute("name")
+                homepage_tags = package_tag.getElementsByTagName("homepage")
+                homepage = homepage_tags[0].firstChild.nodeValue \
+                           if homepage_tags else ""
+                desc_tags = package_tag.getElementsByTagName("description")
+                desc = desc_tags[0].firstChild.nodeValue if desc_tags else ""
 
-        package = None
+                package = self.store_package(cat, pkg, homepage, desc)
 
-        for line in output:
-            match = line_re.match(line)
-
-            if not match:
-                continue
-
-            cpv = match.group('cpv')
-            slot = match.group('slot')
-            overlay = match.group('overlay')
-
-            cat, pkg, ver, rev = portage.catpkgsplit(cpv)
-
-            packages['%s/%s' % (cat, pkg)] = True
-
-            if not package or not \
-               (cat == package.category and pkg == package.name):
-                package = self.store_package(cat, pkg)
-
-            self.store_version(package, cpv, slot, overlay)
+                for version_tag in package_tag.getElementsByTagName("version"):
+                    cpv = "%s/%s-%s" % (cat, pkg,
+                                        version_tag.getAttribute("id"))
+                    slot = version_tag.getAttribute("slot")
+                    overlay = version_tag.getAttribute("overlay")
+                    self.store_version(package, cpv, slot, overlay)
 
         if self.purge_packages and not query:
             for package in Package.objects.all():
-                cp = "%s/%s" % (package.category, package.name)
-                if cp not in packages:
-                    self.logger.info('- [p] %s' % (package))
-                    package.delete()
+                self.logger.info('- [p] %s' % (package))
+                package.delete()
 
-    def store_package(self, cat, pkg):
+    def store_package(self, cat, pkg, homepage, description):
         created = False
         obj = self.cache_get_package(cat, pkg)
 
         if not obj:
             obj, created = Package.objects.get_or_create(
                 category=cat,
-                name=pkg
+                name=pkg,
+                homepage=homepage,
+                description=description,
             )
             self.cache_store_package(obj)
 
@@ -197,7 +188,13 @@ class ScanPortage(object):
                 package=package, slot=slot,
                 revision=rev, version=ver,
                 overlay=overlay,
-                defaults={"alive": True, "packaged": True}
+                defaults={
+                    "alive": True,
+                    "packaged": True,
+                    "version_type": get_version_type(ver),
+                    "confidence": 100,
+                    "handler": "portage"
+                }
             )
         if not created:  # Created objects have defaults values
             obj.alive = True
diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py
index bb34836..5629b7b 100644
--- a/pym/euscan/helpers.py
+++ b/pym/euscan/helpers.py
@@ -44,7 +44,7 @@ def get_version_type(version):
         if token in gentoo_types:
             types.append(token)
     if types:
-        return types[0]
+        return types[0]  # TODO: consider returning all types
     return "release"
 
 

From 699bb372a5591b54ab93d6085fca3b5a851b55fb Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Tue, 26 Jun 2012 17:17:35 +0200
Subject: [PATCH 29/37] euscanwww: Added README for celery

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/README.Celery | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 euscanwww/README.Celery

diff --git a/euscanwww/README.Celery b/euscanwww/README.Celery
new file mode 100644
index 0000000..5159e32
--- /dev/null
+++ b/euscanwww/README.Celery
@@ -0,0 +1,40 @@
+Requirements
+============
+
+Installing requirements with setup.py is the easiest way, just run::
+
+    python setup.py develop
+
+or::
+
+    python setup.py install  # to install euscan and requirements
+
+If you prefer to use portage just install dev-python/django-celery-2.5.5
+
+There's the need of having a broker for tasks. The default and reccommended
+broker is RabbitMQ.
+Install with::
+emerge rabbitmq-server
+
+
+Running Celery
+==============
+
+You'll need:
+* celeryd (celery daemon for running tasks)::
+
+    python manage.py celeryd -E -l INFO
+
+* celerycam (for monitoring celery and see the results in the django admin page)::
+
+    python manage.py celerycam
+
+* celerybeat (for running periodic tasks)::
+
+    python manage.py celerybeat -l INFO
+
+
+TODO
+====
+
+Provide a supervisord configuration

From 8fdac1014ca3173dd655adb9305066290beeed20 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Tue, 26 Jun 2012 18:13:28 +0200
Subject: [PATCH 30/37] euscanwww: Adding infos to EuscanResults

Now we store metadata, output from euscan is cleaned from colors
encoding when outputting json

Signed-off-by: volpino <fox91@anche.no>
---
 ...can_time__add_field_euscanresult_ebuild.py | 191 ++++++++++++++++++
 euscanwww/djeuscan/models.py                  |   3 +
 .../djeuscan/processing/scan_upstream.py      |  17 +-
 pym/euscan/out.py                             |   3 +-
 4 files changed, 207 insertions(+), 7 deletions(-)
 create mode 100644 euscanwww/djeuscan/migrations/0006_auto__add_field_euscanresult_scan_time__add_field_euscanresult_ebuild.py

diff --git a/euscanwww/djeuscan/migrations/0006_auto__add_field_euscanresult_scan_time__add_field_euscanresult_ebuild.py b/euscanwww/djeuscan/migrations/0006_auto__add_field_euscanresult_scan_time__add_field_euscanresult_ebuild.py
new file mode 100644
index 0000000..9e4bcce
--- /dev/null
+++ b/euscanwww/djeuscan/migrations/0006_auto__add_field_euscanresult_scan_time__add_field_euscanresult_ebuild.py
@@ -0,0 +1,191 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'EuscanResult.scan_time'
+        db.add_column('djeuscan_euscanresult', 'scan_time',
+                      self.gf('django.db.models.fields.FloatField')(null=True, blank=True),
+                      keep_default=False)
+
+        # Adding field 'EuscanResult.ebuild'
+        db.add_column('djeuscan_euscanresult', 'ebuild',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=256, blank=True),
+                      keep_default=False)
+
+    def backwards(self, orm):
+        # Deleting field 'EuscanResult.scan_time'
+        db.delete_column('djeuscan_euscanresult', 'scan_time')
+
+        # Deleting field 'EuscanResult.ebuild'
+        db.delete_column('djeuscan_euscanresult', 'ebuild')
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'djeuscan.categoryassociation': {
+            'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.categorylog': {
+            'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.euscanresult': {
+            'Meta': {'object_name': 'EuscanResult'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'djeuscan.herd': {
+            'Meta': {'object_name': 'Herd'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
+            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'djeuscan.herdassociation': {
+            'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.herdlog': {
+            'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.log': {
+            'Meta': {'object_name': 'Log'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'djeuscan.maintainer': {
+            'Meta': {'object_name': 'Maintainer'},
+            'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.maintainerassociation': {
+            'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.maintainerlog': {
+            'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
+        },
+        'djeuscan.package': {
+            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
+            'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.packageassociation': {
+            'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.refreshpackagequery': {
+            'Meta': {'object_name': 'RefreshPackageQuery'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
+        },
+        'djeuscan.version': {
+            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
+            'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'version_type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
+        },
+        'djeuscan.versionlog': {
+            'Meta': {'object_name': 'VersionLog'},
+            'action': ('django.db.models.fields.IntegerField', [], {}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.worldlog': {
+            'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        }
+    }
+
+    complete_apps = ['djeuscan']
\ No newline at end of file
diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py
index 5bca337..e0418d4 100644
--- a/euscanwww/djeuscan/models.py
+++ b/euscanwww/djeuscan/models.py
@@ -187,6 +187,9 @@ class EuscanResult(models.Model):
     datetime = models.DateTimeField()
     result = models.TextField(blank=True)
 
+    scan_time = models.FloatField(null=True, blank=True)
+    ebuild = models.CharField(blank=True, max_length=256)
+
     objects = EuscanResultManager()
 
     class Meta:
diff --git a/euscanwww/djeuscan/processing/scan_upstream.py b/euscanwww/djeuscan/processing/scan_upstream.py
index 2791ce5..045a32c 100644
--- a/euscanwww/djeuscan/processing/scan_upstream.py
+++ b/euscanwww/djeuscan/processing/scan_upstream.py
@@ -25,26 +25,31 @@ class ScanUpstream(object):
 
         try:
             cpv = out[package]["metadata"]["cpv"]
+            scan_time = out[package]["metadata"]["scan_time"]
+            ebuild = out[package]["metadata"]["ebuild"]
         except KeyError:
             return {}
 
-        obj = self.store_package(cpv)
+        with commit_on_success():
+            obj = self.store_package(cpv)
 
-        for res in out[package]["result"]:
-            self.store_version(obj, res["version"], " ".join(res["urls"]))
+            for res in out[package]["result"]:
+                self.store_version(obj, res["version"], " ".join(res["urls"]))
 
-        self.store_result(obj, out_json)
+            self.store_result(obj, out_json, scan_time, ebuild)
 
         return out
 
-    def store_result(self, package, log):
+    def store_result(self, package, formatted_log, scan_time, ebuild):
         # Remove previous logs
         EuscanResult.objects.filter(package=package).delete()
 
         obj = EuscanResult()
         obj.package = package
-        obj.result = log
+        obj.result = formatted_log
         obj.datetime = timezone.now()
+        obj.scan_time = scan_time
+        obj.ebuild = ebuild
         obj.save()
 
     def store_package(self, cpv):
diff --git a/pym/euscan/out.py b/pym/euscan/out.py
index 1a61579..26f5417 100644
--- a/pym/euscan/out.py
+++ b/pym/euscan/out.py
@@ -3,6 +3,7 @@ from collections import defaultdict
 import json
 import signal
 import time
+import re
 
 from gentoolkit import pprinter as pp
 import portage
@@ -61,7 +62,7 @@ class EOutputMem(EOutput):
         self.out = StringIO()
 
     def getvalue(self):
-        return self.out.getvalue()
+        return re.sub("\033\[[0-9;]+m", "", self.out.getvalue())
 
     def _write(self, f, msg):
         super(EOutputMem, self)._write(self.out, msg)

From 1ae2dcba50411d1b33584dfa088e1e8057d5d444 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Tue, 26 Jun 2012 18:31:35 +0200
Subject: [PATCH 31/37] euscanwww: settable options for _run_in_chunks()

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/tasks.py     | 6 +++++-
 euscanwww/euscanwww/settings.py | 3 +++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index 8dfbeaa..1e66838 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -8,6 +8,8 @@ from celery.task import task, periodic_task
 from celery.task.schedules import crontab
 from celery.task.sets import TaskSet
 
+from django.conf import settings
+
 from djeuscan.models import Package, RefreshPackageQuery
 from djeuscan.processing.regen_rrds import regen_rrds
 from djeuscan.processing.update_counters import update_counters
@@ -32,7 +34,9 @@ def _chunks(it, n):
         yield [first] + list(islice(it, n - 1))
 
 
-def _run_in_chunks(task, packages, kwargs=None, concurrently=8, n=32):
+def _run_in_chunks(task, packages, kwargs=None,
+                   concurrently=settings.TASKS_CONCURRENTLY,
+                   n=settings.TASKS_SUBTASK_PACKAGES):
     """
     Launches a TaskSet at a time with <concurrently> subtasks.
     Each subtask has <n> packages to handle
diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py
index 116fcd9..0fe0f89 100644
--- a/euscanwww/euscanwww/settings.py
+++ b/euscanwww/euscanwww/settings.py
@@ -235,6 +235,9 @@ CELERY_RESULT_BACKEND = "amqp"
 BROKER_CONNECTION_TIMEOUT = 3600
 CELERYD_CONCURRENCY = 4
 
+TASKS_CONCURRENTLY = 8
+TASKS_SUBTASK_PACKAGES = 32
+
 # LDAP authentication
 # TODO: Test data - change me!
 AUTH_LDAP_SERVER_URI = "ldap://localhost"

From 951624759a5412b46528363b5e5ca6293cac2676 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Wed, 27 Jun 2012 10:15:30 +0200
Subject: [PATCH 32/37] euscanwww: Fixing tasks

Fixing minor bugs in tasks.py and little related stuff

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/models.py                   |  3 ---
 euscanwww/djeuscan/processing/scan_metadata.py |  7 ++++---
 euscanwww/djeuscan/processing/scan_portage.py  |  2 +-
 euscanwww/djeuscan/processing/scan_upstream.py |  5 +++--
 euscanwww/djeuscan/tasks.py                    | 18 +++++++++---------
 .../admin/djcelery/taskstate/change_list.html  |  2 +-
 6 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py
index e0418d4..e5abd81 100644
--- a/euscanwww/djeuscan/models.py
+++ b/euscanwww/djeuscan/models.py
@@ -272,9 +272,6 @@ class RefreshPackageQuery(models.Model):
     query = models.CharField(max_length=256, unique=True)
     priority = models.IntegerField(default=0)
 
-    class Meta:
-        get_latest_by = "priority"
-
     def __unicode__(self):
         return u'[%d] %s' % (self.priority, self.query)
 
diff --git a/euscanwww/djeuscan/processing/scan_metadata.py b/euscanwww/djeuscan/processing/scan_metadata.py
index 4f06d05..02a8e84 100644
--- a/euscanwww/djeuscan/processing/scan_metadata.py
+++ b/euscanwww/djeuscan/processing/scan_metadata.py
@@ -96,8 +96,8 @@ class ScanMetadata(object):
                         self.style.ERROR("Bad maintainer: '%s' '%s'" % \
                                          (maintainer.name, maintainer.email))
                     )
-
         obj.save()
+        return True
 
     def store_herd(self, name, email):
         if not name:
@@ -142,6 +142,7 @@ def scan_metadata(packages=None, logger=None):
 
     for pkg in packages:
         if isinstance(pkg, Package):
-            scan_handler.scan('%s/%s' % (pkg.category, pkg.name), pkg)
+            result = scan_handler.scan('%s/%s' % (pkg.category, pkg.name), pkg)
         else:
-            scan_handler.scan(pkg)
+            result = scan_handler.scan(pkg)
+    return result
diff --git a/euscanwww/djeuscan/processing/scan_portage.py b/euscanwww/djeuscan/processing/scan_portage.py
index 34456ea..14d7a8e 100644
--- a/euscanwww/djeuscan/processing/scan_portage.py
+++ b/euscanwww/djeuscan/processing/scan_portage.py
@@ -299,5 +299,5 @@ def scan_portage(packages=None, no_log=False, purge_packages=False,
 
     if purge_versions:
         purge_versions(logger=logger, no_log=no_log)
-
     logger.info('Done.')
+    return True
diff --git a/euscanwww/djeuscan/processing/scan_upstream.py b/euscanwww/djeuscan/processing/scan_upstream.py
index 045a32c..827ad91 100644
--- a/euscanwww/djeuscan/processing/scan_upstream.py
+++ b/euscanwww/djeuscan/processing/scan_upstream.py
@@ -131,11 +131,12 @@ def scan_upstream(packages=None, purge_versions=False,
 
     for pkg in packages:
         if isinstance(pkg, Package):
-            scan_handler.scan('%s/%s' % (pkg.category, pkg.name))
+            result = scan_handler.scan('%s/%s' % (pkg.category, pkg.name))
         else:
-            scan_handler.scan(pkg)
+            result = scan_handler.scan(pkg)
 
     if purge_versions:
         purge_versions(logger=logger)
 
     logger.info('Done.')
+    return result
diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index 1e66838..4f6842e 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -114,7 +114,7 @@ def scan_metadata_all_task():
 
 
 @task
-def _scan_portage_task(packages, no_logs=False, purge_packages=False,
+def _scan_portage_task(packages, no_log=False, purge_packages=False,
                        purge_versions=False, prefetch=False):
     """
     Scans portage for the given set of packages
@@ -125,7 +125,7 @@ def _scan_portage_task(packages, no_logs=False, purge_packages=False,
 
     result = scan_portage(
         packages=packages,
-        no_logs=no_logs,
+        no_log=no_log,
         purge_packages=purge_packages,
         purge_versions=purge_versions,
         prefetch=prefetch,
@@ -137,24 +137,24 @@ def _scan_portage_task(packages, no_logs=False, purge_packages=False,
 
 
 @task
-def scan_portage_list_task(query, no_logs=False, purge_packages=False,
+def scan_portage_list_task(query, no_log=False, purge_packages=False,
                            purge_versions=False, prefetch=False):
     """
     Runs a parallel portage scan for packages in the query list (space
     separated string). Task used only from the web interface.
     """
-    kwargs = {"no_logs": no_logs, "purge_packages": purge_packages,
+    kwargs = {"no_log": no_log, "purge_packages": purge_packages,
               "purge_versions": purge_versions, "prefetch": prefetch}
     _run_in_chunks(_scan_portage_task, [p for p in query.split()], kwargs)
 
 
 @task
-def scan_portage_all_task(no_logs=False, purge_packages=False,
+def scan_portage_all_task(no_log=False, purge_packages=False,
                           purge_versions=False, prefetch=False):
     """
     Runs a parallel portage scan for all packages
     """
-    kwargs = {"no_logs": no_logs, "purge_packages": purge_packages,
+    kwargs = {"no_log": no_log, "purge_packages": purge_packages,
               "purge_versions": purge_versions, "prefetch": prefetch}
     _run_in_chunks(_scan_metadata_task, Package.objects.all(), kwargs)
 
@@ -251,8 +251,8 @@ def consume_refresh_package_request():
     Satisfies user requests for package refreshing, runs every minute
     """
     try:
-        obj = RefreshPackageQuery.objects.latest()
-    except RefreshPackageQuery.DoesNotExist:
+        obj = RefreshPackageQuery.objects.all().order_by('-priority')[0]
+    except IndexError:
         return {}
     else:
         result = scan_package_task(obj.query)
@@ -277,7 +277,7 @@ admin_tasks = [
     scan_portage_list_task,
     scan_upstream_all_task,
     scan_upstream_list_task,
-    update_portage_trees,
+    update_portage_trees_task,
     update_task,
     scan_package_task,
 ]
diff --git a/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html b/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html
index 0076a6c..db2680b 100644
--- a/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html
+++ b/euscanwww/djeuscan/templates/admin/djcelery/taskstate/change_list.html
@@ -73,7 +73,7 @@
             });
         });
 
-        setTimeout(location.reload, 10000);
+        setTimeout(location.reload, 30000);
 
       });
     })(django.jQuery);

From b06965f386b3e3410a252d1076d95887ca06f93d Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Thu, 28 Jun 2012 11:08:05 +0200
Subject: [PATCH 33/37] euscan: plugin system for handlers

Signed-off-by: volpino <fox91@anche.no>
---
 pym/euscan/handlers/__init__.py | 15 +++++++++++++--
 pym/euscan/handlers/cpan.py     |  1 +
 pym/euscan/handlers/generic.py  |  1 +
 pym/euscan/handlers/github.py   |  8 +++++++-
 pym/euscan/handlers/kde.py      |  2 ++
 pym/euscan/handlers/php.py      |  1 +
 pym/euscan/handlers/pypi.py     |  1 +
 pym/euscan/handlers/rubygem.py  |  1 +
 8 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/pym/euscan/handlers/__init__.py b/pym/euscan/handlers/__init__.py
index 0b4b510..bcbd680 100644
--- a/pym/euscan/handlers/__init__.py
+++ b/pym/euscan/handlers/__init__.py
@@ -1,6 +1,17 @@
-from euscan.handlers import generic, php, pypi, rubygem, kde, cpan, github
+import pkgutil
 
-handlers = [kde, php, pypi, rubygem, cpan, github, generic]
+# autoimport all modules in this directory and append them to handlers list
+handlers = []
+for loader, module_name, is_pkg in  pkgutil.walk_packages(__path__):
+    module = loader.find_module(module_name).load_module(module_name)
+    handlers.append(module)
+
+# sort handlers by priority (e.g.: generic should be run lastly)
+handlers = sorted(
+    handlers,
+    key=lambda handler: handler.PRIORITY,
+    reverse=True
+)
 
 
 def find_best_handler(cpv, url):
diff --git a/pym/euscan/handlers/cpan.py b/pym/euscan/handlers/cpan.py
index 7dcc246..7115c20 100644
--- a/pym/euscan/handlers/cpan.py
+++ b/pym/euscan/handlers/cpan.py
@@ -7,6 +7,7 @@ from euscan import helpers, output
 
 HANDLER_NAME = "cpan"
 CONFIDENCE = 100.0
+PRIORITY = 100
 
 _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*")
 
diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/generic.py
index 92c2984..5c6d467 100644
--- a/pym/euscan/handlers/generic.py
+++ b/pym/euscan/handlers/generic.py
@@ -14,6 +14,7 @@ from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \
 
 HANDLER_NAME = "generic"
 CONFIDENCE = 50.0
+PRIORITY = 0
 
 BRUTEFORCE_HANDLER_NAME = "brute_force"
 BRUTEFORCE_CONFIDENCE = 30.0
diff --git a/pym/euscan/handlers/github.py b/pym/euscan/handlers/github.py
index 10386fd..920e6ef 100644
--- a/pym/euscan/handlers/github.py
+++ b/pym/euscan/handlers/github.py
@@ -1,4 +1,5 @@
-import json, urllib2
+import json
+import urllib2
 import re
 
 import portage
@@ -7,16 +8,20 @@ from euscan import helpers, output
 
 HANDLER_NAME = "github"
 CONFIDENCE = 100.0
+PRIORITY = 100
+
 
 def can_handle(cpv, url):
     return url.startswith('mirror://github/')
 
+
 def guess_package(cp, url):
     match = re.search('^mirror://github/(.*?)/(.*?)/(.*)$', url)
 
     assert(match)
     return (match.group(1), match.group(2), match.group(3))
 
+
 def scan(cpv, url):
     'http://developer.github.com/v3/repos/downloads/'
 
@@ -46,5 +51,6 @@ def scan(cpv, url):
                 continue
             yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE)
 
+
 def brute_force(cpv, url):
     return []
diff --git a/pym/euscan/handlers/kde.py b/pym/euscan/handlers/kde.py
index 2b27639..406b079 100644
--- a/pym/euscan/handlers/kde.py
+++ b/pym/euscan/handlers/kde.py
@@ -1,5 +1,7 @@
 from euscan.handlers import generic
 
+PRIORITY = 100
+
 HANDLER_NAME = "kde"
 
 
diff --git a/pym/euscan/handlers/php.py b/pym/euscan/handlers/php.py
index 82c49c5..158280c 100644
--- a/pym/euscan/handlers/php.py
+++ b/pym/euscan/handlers/php.py
@@ -7,6 +7,7 @@ from euscan import helpers, output
 
 HANDLER_NAME = "php"
 CONFIDENCE = 100.0
+PRIORITY = 100
 
 
 def can_handle(cpv, url):
diff --git a/pym/euscan/handlers/pypi.py b/pym/euscan/handlers/pypi.py
index 0b3ed73..1dcad99 100644
--- a/pym/euscan/handlers/pypi.py
+++ b/pym/euscan/handlers/pypi.py
@@ -7,6 +7,7 @@ from euscan import helpers, output
 
 HANDLER_NAME = "pypi"
 CONFIDENCE = 100.0
+PRIORITY = 100
 
 
 def can_handle(cpv, url):
diff --git a/pym/euscan/handlers/rubygem.py b/pym/euscan/handlers/rubygem.py
index 8bbd5ab..c0f7ade 100644
--- a/pym/euscan/handlers/rubygem.py
+++ b/pym/euscan/handlers/rubygem.py
@@ -7,6 +7,7 @@ from euscan import helpers, output
 
 HANDLER_NAME = "rubygem"
 CONFIDENCE = 100.0
+PRIORITY = 100
 
 
 def can_handle(cpv, url):

From 0d6376681f72b78fc43e7861586f76039a9c722a Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Thu, 28 Jun 2012 12:20:57 +0200
Subject: [PATCH 34/37] euscan: fixed progressbar

added -p option, now the progressbar is "global" and shows the total
progress

Signed-off-by: volpino <fox91@anche.no>
---
 bin/euscan             | 43 +++++++++++++++++++++++++-----------------
 pym/euscan/__init__.py |  3 ++-
 pym/euscan/out.py      | 15 ++++++++++-----
 pym/euscan/scan.py     | 36 ++++++++++++-----------------------
 4 files changed, 50 insertions(+), 47 deletions(-)

diff --git a/bin/euscan b/bin/euscan
index 7bea5bd..89f61bf 100755
--- a/bin/euscan
+++ b/bin/euscan
@@ -21,8 +21,8 @@ __description__ = "A tool to detect new upstream releases."
 import sys
 import os
 import getopt
-import errno
-import httplib
+from errno import EINTR, EINVAL
+from httplib import HTTPConnection
 
 from portage.output import white, yellow, turquoise, green
 from portage.exception import AmbiguousPackageName
@@ -37,6 +37,8 @@ from euscan.out import progress_bar
 
 
 # Globals
+isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty()
+
 
 def exit_helper(status):
     if CONFIG["format"]:
@@ -52,7 +54,7 @@ def setup_signals():
         signal.signal(signal.SIGINT, signal.SIG_IGN)
         signal.signal(signal.SIGTERM, signal.SIG_IGN)
         print()
-        exit_helper(errno.EINTR)
+        exit_helper(EINTR)
 
     signal.signal(signal.SIGINT, exithandler)
     signal.signal(signal.SIGTERM, exithandler)
@@ -121,6 +123,8 @@ def print_usage(_error=None, help=None):
         print(yellow(" -f, --format=<format>") +
               "     - define the output " + yellow("<format>") +
               " (available: json)", file=out)
+        print(yellow(" -p, --progress") +
+              "     - display a progress bar", file=out)
         print(file=out)
 
     if _error in ('packages',) or help:
@@ -172,6 +176,8 @@ def parse_args():
                 CONFIG['format'] = a
                 CONFIG['nocolor'] = True
                 pp.output.nocolor()
+            elif o in ("-p", "--progress"):
+                CONFIG['progress'] = isatty
             else:
                 return_code = False
 
@@ -179,7 +185,7 @@ def parse_args():
 
     # here are the different allowed command line options (getopt args)
     getopt_options = {'short': {}, 'long': {}}
-    getopt_options['short']['global'] = "hVCqv1bf:"
+    getopt_options['short']['global'] = "hVCqv1bf:p"
     getopt_options['long']['global'] = [
         "help", "version", "nocolor", "quiet", "verbose", "oneshot",
         "brute-force=", "format="
@@ -207,7 +213,7 @@ def parse_args():
 def main():
     """Parse command line and execute all actions."""
     CONFIG['nocolor'] = (
-        port_settings["NOCOLOR"] in ('yes', 'true') or not sys.stdout.isatty()
+        port_settings["NOCOLOR"] in ('yes', 'true') or not isatty
     )
     if CONFIG['nocolor']:
         pp.output.nocolor()
@@ -230,19 +236,22 @@ def main():
 
         else:
             print_usage(e.value)
-            exit_helper(errno.EINVAL)
+            exit_helper(EINVAL)
 
     if CONFIG['verbose'] > 2:
-        httplib.HTTPConnection.debuglevel = 1
+        HTTPConnection.debuglevel = 1
 
-    isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty()
+    if not CONFIG["format"]:
+        CONFIG["progress"] = False
+
+    on_progress = None
+    if CONFIG['progress']:
+        on_progress_gen = progress_bar()
+        on_progress = on_progress_gen.next()
+        on_progress(maxval=len(queries) * 100, increment=0)
 
     for query in queries:
-        on_progress = None
-        if (CONFIG['format'] or CONFIG['quiet']) and isatty:
-            print("%s:" % query, file=sys.stderr)
-            on_progress_gen = progress_bar()
-            on_progress = on_progress_gen.next()
+        on_progress(increment=10, label=query)
 
         ret = []
 
@@ -271,16 +280,16 @@ def main():
             output.eerror('%s: %s' % (query, str(err)))
             exit_helper(1)
 
-        if (CONFIG['format'] or CONFIG['quiet']) and isatty:
-            on_progress_gen.next()
-            print("\n", file=sys.stderr)
-
         if not ret and not CONFIG['quiet']:
             output.ewarn(
                 "Didn't find any new version, check package's homepage " +
                 "for more informations"
             )
 
+    if CONFIG['progress']:
+        on_progress_gen.next()
+        print("\n", file=sys.stderr)
+
     output.set_query(None)
 
 
diff --git a/pym/euscan/__init__.py b/pym/euscan/__init__.py
index 84754ee..9c125be 100644
--- a/pym/euscan/__init__.py
+++ b/pym/euscan/__init__.py
@@ -19,7 +19,8 @@ CONFIG = {
     'skip-robots-txt': False,
     'cache': False,
     'format': None,
-    'indent': 2
+    'indent': 2,
+    'progress': False
 }
 
 BLACKLIST_VERSIONS = [
diff --git a/pym/euscan/out.py b/pym/euscan/out.py
index 26f5417..1928980 100644
--- a/pym/euscan/out.py
+++ b/pym/euscan/out.py
@@ -11,15 +11,20 @@ from portage.output import EOutput, TermProgressBar
 
 
 class ProgressHandler(object):
-    def __init__(self):
+    def __init__(self, progress_bar):
         self.curval = 0
         self.maxval = 0
         self.last_update = 0
         self.min_display_latency = 0.2
+        self.progress_bar = progress_bar
+
+    def on_progress(self, maxval=None, increment=1, label=None):
+        self.maxval = maxval or self.maxval
+        self.curval += increment
+
+        if label:
+            self.progress_bar.label(label)
 
-    def on_progress(self, maxval, curval):
-        self.maxval = maxval
-        self.curval = curval
         cur_time = time.time()
         if cur_time - self.last_update >= self.min_display_latency:
             self.last_update = cur_time
@@ -33,7 +38,7 @@ def progress_bar():
     on_progress = None
     progress_bar = TermProgressBar()
 
-    progress_handler = ProgressHandler()
+    progress_handler = ProgressHandler(progress_bar)
     on_progress = progress_handler.on_progress
 
     def display():
diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py
index 2ac828f..be49b96 100644
--- a/pym/euscan/scan.py
+++ b/pym/euscan/scan.py
@@ -45,15 +45,17 @@ def filter_versions(cp, versions):
 def scan_upstream_urls(cpv, urls, on_progress):
     versions = []
 
-    maxval = len(urls) + 5
-    curval = 1
+    progress_available = 70
+    num_urls = sum([len(urls[fn]) for fn in urls])
+    progress_increment = progress_available / num_urls
 
     for filename in urls:
-        curval += 1
-        if on_progress:
-            on_progress(maxval, curval)
-
         for url in urls[filename]:
+
+            if on_progress and progress_available > 0:
+                on_progress(increment=progress_increment)
+                progress_available -= progress_increment
+
             if not CONFIG['quiet'] and not CONFIG['format']:
                 pp.uprint()
             output.einfo("SRC_URI is '%s'" % url)
@@ -82,15 +84,10 @@ def scan_upstream_urls(cpv, urls, on_progress):
 
     cp, ver, rev = portage.pkgsplit(cpv)
 
-    curval += 1
-    if on_progress:
-        on_progress(maxval, curval)
-
     result = filter_versions(cp, versions)
 
-    curval += 1
-    if on_progress:
-        on_progress(maxval, curval)
+    if on_progress and progress_available > 0:
+        on_progress(increment=progress_available)
 
     return result
 
@@ -118,9 +115,6 @@ def scan_upstream(query, on_progress=None):
     Scans the upstream searching new versions for the given query
     """
 
-    maxval = 3
-    curval = 0
-
     matches = []
 
     if query.endswith(".ebuild"):
@@ -159,9 +153,8 @@ def scan_upstream(query, on_progress=None):
     output.metadata("cp", pkg.cp, show=False)
     output.metadata("cpv", pkg.cpv, show=False)
 
-    curval += 1
     if on_progress:
-        on_progress(maxval, curval)
+        on_progress(increment=10)
 
     if pkg.cp in BLACKLIST_PACKAGES:
         output.ewarn(
@@ -213,15 +206,10 @@ def scan_upstream(query, on_progress=None):
     scan_time = (datetime.now() - start_time).total_seconds()
     output.metadata("scan_time", scan_time, show=False)
 
-    curval += 1
-    if on_progress:
-        on_progress(maxval, curval)
-
     result = scan_upstream_urls(pkg.cpv, urls, on_progress)
 
-    curval += 1
     if on_progress:
-        on_progress(maxval, curval)
+        on_progress(increment=10)
 
     if len(result) > 0:
         if not (CONFIG['format'] or CONFIG['quiet']):

From d7bf130fdae66fa38118c07d2d0e27942e0982a2 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Thu, 28 Jun 2012 12:43:46 +0200
Subject: [PATCH 35/37] euscan: minor fixes

Signed-off-by: volpino <fox91@anche.no>
---
 bin/euscan                    | 3 ++-
 pym/euscan/handlers/github.py | 8 +++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/bin/euscan b/bin/euscan
index 89f61bf..963193f 100755
--- a/bin/euscan
+++ b/bin/euscan
@@ -251,7 +251,8 @@ def main():
         on_progress(maxval=len(queries) * 100, increment=0)
 
     for query in queries:
-        on_progress(increment=10, label=query)
+        if CONFIG["progress"]:
+            on_progress(increment=10, label=query)
 
         ret = []
 
diff --git a/pym/euscan/handlers/github.py b/pym/euscan/handlers/github.py
index 920e6ef..ac5baca 100644
--- a/pym/euscan/handlers/github.py
+++ b/pym/euscan/handlers/github.py
@@ -35,11 +35,13 @@ def scan(cpv, url):
     # now create a filename-matching regexp
     # XXX: supposedly replace first with (?P<foo>...)
     # and remaining ones with (?P=foo)
-    fnre = re.compile('^%s$' % re.escape(filename).replace(re.escape(ver), '(.*?)'))
+    fnre = re.compile('^%s$' % \
+                      re.escape(filename).replace(re.escape(ver), '(.*?)'))
 
-    output.einfo("Using github API for: " + '/'.join(package))
+    output.einfo("Using github API for: " + '/'.join(filename))
 
-    dlreq = urllib2.urlopen('https://api.github.com/repos/%s/%s/downloads' % (user, project))
+    dlreq = urllib2.urlopen('https://api.github.com/repos/%s/%s/downloads' % \
+                            (user, project))
     dls = json.load(dlreq)
 
     for dl in dls:

From e9c03fbded1cd8beef76680450e5b818cd3a161f Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Fri, 29 Jun 2012 14:13:25 +0200
Subject: [PATCH 36/37] euscanwww: Fixed CSS stylesheet

Added Twitter Bootstrap and tweaked the layout to use the grid layout

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/templates/_base.html       | 105 +++++++------
 .../djeuscan/templates/euscan/_datatable.html |   2 +-
 .../djeuscan/templates/euscan/package.html    |   6 +-
 .../djeuscan/templates/euscan/statistics.html |   1 -
 .../templates/registration/login.html         |  16 +-
 .../registration/registration_form.html       |   2 +-
 euscanwww/htdocs/css/bootstrap.min.css        |   9 ++
 euscanwww/htdocs/css/style.css                | 146 ++++++------------
 8 files changed, 133 insertions(+), 154 deletions(-)
 create mode 100644 euscanwww/htdocs/css/bootstrap.min.css

diff --git a/euscanwww/djeuscan/templates/_base.html b/euscanwww/djeuscan/templates/_base.html
index da71b71..717eb20 100644
--- a/euscanwww/djeuscan/templates/_base.html
+++ b/euscanwww/djeuscan/templates/_base.html
@@ -10,6 +10,7 @@
     <link rel="alternate" type="application/atom+xml" title="Global log" href="{% url "global_feed" %}" />
     {% endblock %}
     {% block css %}
+    <link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/bootstrap.min.css" media="screen" title="Normal" />
     <link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/style.css" media="screen" title="Normal" />
     {% endblock %}
     {% block javascript %}
@@ -22,53 +23,65 @@
       </a>
       {% block header %}<h1>Ebuild Upstream Scanner (euscan)</h1>{% endblock %}
     </div>
-    <div id="content">
-      {% block content %}{% endblock %}
-    </div>
-    <div id="menus">
-      {% block menus %}
-        <div class="menu">
-          <ul>
-            {% block menu %}
-              <li><a href="{% url "index" %}">Home</a></li>
-              <li><a href="{% url "categories" %}">Categories</a></li>
-              <li><a href="{% url "herds" %}">Herds</a></li>
-              <li><a href="{% url "maintainers" %}">Maintainers</a></li>
-              <li><a href="{% url "overlays" %}">Overlays</a></li>
-              <li><a href="{% url "world" %}">Scan World</a></li>
-              <li><a href="{% url "statistics" %}">Statistics</a></li>
-
-              <li>---</li>
-
-              {% if user.is_authenticated %}
-                <li><a href="{% url "accounts_index" %}">{{ user }}'s profile</a></li>
-                <ul class="submenu">
-                  <li><a href="{% url "accounts_categories" %}">Categories</a></li>
-                  <li><a href="{% url "accounts_herds" %}">Herds</a></li>
-                  <li><a href="{% url "accounts_maintainers" %}">Maintainers</a></li>
-                  <li><a href="{% url "accounts_packages" %}">Packages</a></li>
-                </ul>
-                <li><a href="{% url "django.contrib.auth.views.logout" %}">Logout</a></li>
-              {% else %}
-                <li><a href="{% url "django.contrib.auth.views.login" %}?next={% url "accounts_index" %}">Login</a></li>
-                <li><a href="{% url "registration.views.register" %}">Register</a></li>
-              {% endif %}
-
-              <li>---</li>
-
-              {% block menu_feed %}
-                <li>
-                  <img src="{{ STATIC_URL }}img/feed.png" alt="feed" />
-                  <a title="Global Feed" href="{% url "global_feed" %}">Global Feed</a>
-                </li>
-              {% endblock %}
-              <li>---</li>
-              <li><a href="{% url "api" %}">API</a></li>
-              <li><a href="{% url "about" %}">About</a></li>
-            {% endblock %}
-          </ul>
+    <div class="row-fluid">
+      <div class="span10">
+        <div class="row-fluid">
+          <div class="span1"></div>
+          <div id="content" class="rounded span11">
+            {% block content %}{% endblock %}
+          </div>
         </div>
-      {% endblock %}
+      </div>
+      <div class="span2">
+        <div class="row-fluid">
+          <div id="menus" class="span11">
+            {% block menus %}
+              <div class="menu rounded">
+                <ul>
+                  {% block menu %}
+                    <li><a href="{% url "index" %}">Home</a></li>
+                    <li><a href="{% url "categories" %}">Categories</a></li>
+                    <li><a href="{% url "herds" %}">Herds</a></li>
+                    <li><a href="{% url "maintainers" %}">Maintainers</a></li>
+                    <li><a href="{% url "overlays" %}">Overlays</a></li>
+                    <li><a href="{% url "world" %}">Scan World</a></li>
+                    <li><a href="{% url "statistics" %}">Statistics</a></li>
+
+                    <li>---</li>
+
+                    {% if user.is_authenticated %}
+                      <li><a href="{% url "accounts_index" %}">{{ user }}'s profile</a></li>
+                      <ul class="submenu">
+                        <li><a href="{% url "accounts_categories" %}">Categories</a></li>
+                        <li><a href="{% url "accounts_herds" %}">Herds</a></li>
+                        <li><a href="{% url "accounts_maintainers" %}">Maintainers</a></li>
+                        <li><a href="{% url "accounts_packages" %}">Packages</a></li>
+                      </ul>
+                      <li><a href="{% url "django.contrib.auth.views.logout" %}">Logout</a></li>
+                    {% else %}
+                      <li><a href="{% url "django.contrib.auth.views.login" %}?next={% url "accounts_index" %}">Login</a></li>
+                      <li><a href="{% url "registration.views.register" %}">Register</a></li>
+                    {% endif %}
+
+                    <li>---</li>
+
+                    {% block menu_feed %}
+                      <li>
+                        <img src="{{ STATIC_URL }}img/feed.png" alt="feed" />
+                        <a title="Global Feed" href="{% url "global_feed" %}">Global Feed</a>
+                      </li>
+                    {% endblock %}
+                    <li>---</li>
+                    <li><a href="{% url "api" %}">API</a></li>
+                    <li><a href="{% url "about" %}">About</a></li>
+                  {% endblock %}
+                </ul>
+              </div>
+            {% endblock %}
+          </div>
+          <div class="span1"></div>
+        </div>
+      </div>
     </div>
     <div id="footer">
       <p>
diff --git a/euscanwww/djeuscan/templates/euscan/_datatable.html b/euscanwww/djeuscan/templates/euscan/_datatable.html
index 32163cf..6355be1 100644
--- a/euscanwww/djeuscan/templates/euscan/_datatable.html
+++ b/euscanwww/djeuscan/templates/euscan/_datatable.html
@@ -24,7 +24,7 @@
 
 {% block menus %}
 {{ block.super }}
-<div class="menu">
+<div class="menu notfirst-menu rounded">
   <ul>
     <li><img src="{{ STATIC_URL }}img/gentoo-icon.png" /> Gentoo</li>
     <li><img src="{{ STATIC_URL }}img/overlay-icon.png" /> Overlays</li>
diff --git a/euscanwww/djeuscan/templates/euscan/package.html b/euscanwww/djeuscan/templates/euscan/package.html
index 7b3eaec..57a5c88 100644
--- a/euscanwww/djeuscan/templates/euscan/package.html
+++ b/euscanwww/djeuscan/templates/euscan/package.html
@@ -28,13 +28,13 @@
   {{ package.category }}/{{ package.name }}
 
   {% if user.is_authenticated %}
-    <button class="refresh-button" data-package="{{ package.category }}/{{ package.name }}">
+    <button class="btn refresh-button" data-package="{{ package.category }}/{{ package.name }}">
       Refresh
     </button>
-    <button class="favourite-button {% if favourited %}hide{% endif %}" data-url="{% url "favourite_package" package.category package.name %}">
+    <button class="btn favourite-button {% if favourited %}hide{% endif %}" data-url="{% url "favourite_package" package.category package.name %}">
       Add to favourites
     </button>
-    <button class="unfavourite-button {% if not favourited %}hide{% endif %}" data-url="{% url "unfavourite_package" package.category package.name %}">
+    <button class="btn unfavourite-button {% if not favourited %}hide{% endif %}" data-url="{% url "unfavourite_package" package.category package.name %}">
       Remove from favourites
     </button>
   {% endif %}
diff --git a/euscanwww/djeuscan/templates/euscan/statistics.html b/euscanwww/djeuscan/templates/euscan/statistics.html
index 708603d..d895a44 100644
--- a/euscanwww/djeuscan/templates/euscan/statistics.html
+++ b/euscanwww/djeuscan/templates/euscan/statistics.html
@@ -3,7 +3,6 @@
 {% load url from future %}
 
 {% block content %}
-<h2>What's euscan ?</h2>
 <h2>Statistics</h2>
 
 <h3>Current statistics</h3>
diff --git a/euscanwww/djeuscan/templates/registration/login.html b/euscanwww/djeuscan/templates/registration/login.html
index eaa7c54..c1bf3ae 100644
--- a/euscanwww/djeuscan/templates/registration/login.html
+++ b/euscanwww/djeuscan/templates/registration/login.html
@@ -22,14 +22,20 @@
       <tr>
         <td></td>
         <td>
-          <input type="submit" value="Login" />
+          <input class="btn pull-right" type="submit" value="Login" />
+        </td>
+      </tr>
+      <tr>
+        <td>&nbsp;</td>
+      </tr>
+      <tr>
+        <td>
+        </td>
+        <td>
+          <a class="pull-right" href="{% url "django.contrib.auth.views.password_reset" %}">Forgot password</a>
         </td>
       </tr>
     </table>
     <input type="hidden" name="next" value="/" />
   </form>
-
-  <a href="{% url "registration.views.register" %}">Register</a>
-  <a href="{% url "django.contrib.auth.views.password_reset" %}">Forgot password</a>
-
 {% endblock %}
diff --git a/euscanwww/djeuscan/templates/registration/registration_form.html b/euscanwww/djeuscan/templates/registration/registration_form.html
index c9ae8e4..a96427b 100644
--- a/euscanwww/djeuscan/templates/registration/registration_form.html
+++ b/euscanwww/djeuscan/templates/registration/registration_form.html
@@ -11,7 +11,7 @@
       <tr>
         <td></td>
         <td>
-          <input type="submit" value="Send activation email" />
+          <input class="btn pull-right" type="submit" value="Send activation email" />
         </td>
       </tr>
     </form>
diff --git a/euscanwww/htdocs/css/bootstrap.min.css b/euscanwww/htdocs/css/bootstrap.min.css
new file mode 100644
index 0000000..b74b454
--- /dev/null
+++ b/euscanwww/htdocs/css/bootstrap.min.css
@@ -0,0 +1,9 @@
+/*!
+ * Bootstrap v2.0.4
+ *
+ * Copyright 2012 Twitter, Inc
+ * Licensed under the Apache License v2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Designed and built with all the love in the world @twitter by @mdo and @fat.
+ */article,aside,details,figcaption,figure,footer,header,hgroup,nav,section{display:block}audio,canvas,video{display:inline-block;*display:inline;*zoom:1}audio:not([controls]){display:none}html{font-size:100%;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}a:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}a:hover,a:active{outline:0}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sup{top:-0.5em}sub{bottom:-0.25em}img{max-width:100%;vertical-align:middle;border:0;-ms-interpolation-mode:bicubic}#map_canvas img{max-width:none}button,input,select,textarea{margin:0;font-size:100%;vertical-align:middle}button,input{*overflow:visible;line-height:normal}button::-moz-focus-inner,input::-moz-focus-inner{padding:0;border:0}button,input[type="button"],input[type="reset"],input[type="submit"]{cursor:pointer;-webkit-appearance:button}input[type="search"]{-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;-webkit-appearance:textfield}input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button{-webkit-appearance:none}textarea{overflow:auto;vertical-align:top}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;content:""}.clearfix:after{clear:both}.hide-text{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:28px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;-ms-box-sizing:border-box;box-sizing:border-box}body{margin:0;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:13px;line-height:18px;color:#333;background-color:#fff}a{color:#08c;text-decoration:none}a:hover{color:#005580;text-decoration:underline}.row{margin-left:-20px;*zoom:1}.row:before,.row:after{display:table;content:""}.row:after{clear:both}[class*="span"]{float:left;margin-left:20px}.container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:940px}.span12{width:940px}.span11{width:860px}.span10{width:780px}.span9{width:700px}.span8{width:620px}.span7{width:540px}.span6{width:460px}.span5{width:380px}.span4{width:300px}.span3{width:220px}.span2{width:140px}.span1{width:60px}.offset12{margin-left:980px}.offset11{margin-left:900px}.offset10{margin-left:820px}.offset9{margin-left:740px}.offset8{margin-left:660px}.offset7{margin-left:580px}.offset6{margin-left:500px}.offset5{margin-left:420px}.offset4{margin-left:340px}.offset3{margin-left:260px}.offset2{margin-left:180px}.offset1{margin-left:100px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:28px;margin-left:2.127659574%;*margin-left:2.0744680846382977%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;-ms-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .span12{width:99.99999998999999%;*width:99.94680850063828%}.row-fluid .span11{width:91.489361693%;*width:91.4361702036383%}.row-fluid .span10{width:82.97872339599999%;*width:82.92553190663828%}.row-fluid .span9{width:74.468085099%;*width:74.4148936096383%}.row-fluid .span8{width:65.95744680199999%;*width:65.90425531263828%}.row-fluid .span7{width:57.446808505%;*width:57.3936170156383%}.row-fluid .span6{width:48.93617020799999%;*width:48.88297871863829%}.row-fluid .span5{width:40.425531911%;*width:40.3723404216383%}.row-fluid .span4{width:31.914893614%;*width:31.8617021246383%}.row-fluid .span3{width:23.404255317%;*width:23.3510638276383%}.row-fluid .span2{width:14.89361702%;*width:14.8404255306383%}.row-fluid .span1{width:6.382978723%;*width:6.329787233638298%}.container{margin-right:auto;margin-left:auto;*zoom:1}.container:before,.container:after{display:table;content:""}.container:after{clear:both}.container-fluid{padding-right:20px;padding-left:20px;*zoom:1}.container-fluid:before,.container-fluid:after{display:table;content:""}.container-fluid:after{clear:both}p{margin:0 0 9px}p small{font-size:11px;color:#999}.lead{margin-bottom:18px;font-size:20px;font-weight:200;line-height:27px}h1,h2,h3,h4,h5,h6{margin:0;font-family:inherit;font-weight:bold;color:inherit;text-rendering:optimizelegibility}h1 small,h2 small,h3 small,h4 small,h5 small,h6 small{font-weight:normal;color:#999}h1{font-size:30px;line-height:36px}h1 small{font-size:18px}h2{font-size:24px;line-height:36px}h2 small{font-size:18px}h3{font-size:18px;line-height:27px}h3 small{font-size:14px}h4,h5,h6{line-height:18px}h4{font-size:14px}h4 small{font-size:12px}h5{font-size:12px}h6{font-size:11px;color:#999;text-transform:uppercase}.page-header{padding-bottom:17px;margin:18px 0;border-bottom:1px solid #eee}.page-header h1{line-height:1}ul,ol{padding:0;margin:0 0 9px 25px}ul ul,ul ol,ol ol,ol ul{margin-bottom:0}ul{list-style:disc}ol{list-style:decimal}li{line-height:18px}ul.unstyled,ol.unstyled{margin-left:0;list-style:none}dl{margin-bottom:18px}dt,dd{line-height:18px}dt{font-weight:bold;line-height:17px}dd{margin-left:9px}.dl-horizontal dt{float:left;width:120px;overflow:hidden;clear:left;text-align:right;text-overflow:ellipsis;white-space:nowrap}.dl-horizontal dd{margin-left:130px}hr{margin:18px 0;border:0;border-top:1px solid #eee;border-bottom:1px solid #fff}strong{font-weight:bold}em{font-style:italic}.muted{color:#999}abbr[title]{cursor:help;border-bottom:1px dotted #999}abbr.initialism{font-size:90%;text-transform:uppercase}blockquote{padding:0 0 0 15px;margin:0 0 18px;border-left:5px solid #eee}blockquote p{margin-bottom:0;font-size:16px;font-weight:300;line-height:22.5px}blockquote small{display:block;line-height:18px;color:#999}blockquote small:before{content:'\2014 \00A0'}blockquote.pull-right{float:right;padding-right:15px;padding-left:0;border-right:5px solid #eee;border-left:0}blockquote.pull-right p,blockquote.pull-right small{text-align:right}q:before,q:after,blockquote:before,blockquote:after{content:""}address{display:block;margin-bottom:18px;font-style:normal;line-height:18px}small{font-size:100%}cite{font-style:normal}code,pre{padding:0 3px 2px;font-family:Menlo,Monaco,Consolas,"Courier New",monospace;font-size:12px;color:#333;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}code{padding:2px 4px;color:#d14;background-color:#f7f7f9;border:1px solid #e1e1e8}pre{display:block;padding:8.5px;margin:0 0 9px;font-size:12.025px;line-height:18px;word-break:break-all;word-wrap:break-word;white-space:pre;white-space:pre-wrap;background-color:#f5f5f5;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.15);-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}pre.prettyprint{margin-bottom:18px}pre code{padding:0;color:inherit;background-color:transparent;border:0}.pre-scrollable{max-height:340px;overflow-y:scroll}form{margin:0 0 18px}fieldset{padding:0;margin:0;border:0}legend{display:block;width:100%;padding:0;margin-bottom:27px;font-size:19.5px;line-height:36px;color:#333;border:0;border-bottom:1px solid #e5e5e5}legend small{font-size:13.5px;color:#999}label,input,button,select,textarea{font-size:13px;font-weight:normal;line-height:18px}input,button,select,textarea{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif}label{display:block;margin-bottom:5px}select,textarea,input[type="text"],input[type="password"],input[type="datetime"],input[type="datetime-local"],input[type="date"],input[type="month"],input[type="time"],input[type="week"],input[type="number"],input[type="email"],input[type="url"],input[type="search"],input[type="tel"],input[type="color"],.uneditable-input{display:inline-block;height:18px;padding:4px;margin-bottom:9px;font-size:13px;line-height:18px;color:#555}input,textarea{width:210px}textarea{height:auto}textarea,input[type="text"],input[type="password"],input[type="datetime"],input[type="datetime-local"],input[type="date"],input[type="month"],input[type="time"],input[type="week"],input[type="number"],input[type="email"],input[type="url"],input[type="search"],input[type="tel"],input[type="color"],.uneditable-input{background-color:#fff;border:1px solid #ccc;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-webkit-transition:border linear .2s,box-shadow linear .2s;-moz-transition:border linear .2s,box-shadow linear .2s;-ms-transition:border linear .2s,box-shadow linear .2s;-o-transition:border linear .2s,box-shadow linear .2s;transition:border linear .2s,box-shadow linear .2s}textarea:focus,input[type="text"]:focus,input[type="password"]:focus,input[type="datetime"]:focus,input[type="datetime-local"]:focus,input[type="date"]:focus,input[type="month"]:focus,input[type="time"]:focus,input[type="week"]:focus,input[type="number"]:focus,input[type="email"]:focus,input[type="url"]:focus,input[type="search"]:focus,input[type="tel"]:focus,input[type="color"]:focus,.uneditable-input:focus{border-color:rgba(82,168,236,0.8);outline:0;outline:thin dotted \9;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6)}input[type="radio"],input[type="checkbox"]{margin:3px 0;*margin-top:0;line-height:normal;cursor:pointer}input[type="submit"],input[type="reset"],input[type="button"],input[type="radio"],input[type="checkbox"]{width:auto}.uneditable-textarea{width:auto;height:auto}select,input[type="file"]{height:28px;*margin-top:4px;line-height:28px}select{width:220px;border:1px solid #bbb}select[multiple],select[size]{height:auto}select:focus,input[type="file"]:focus,input[type="radio"]:focus,input[type="checkbox"]:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.radio,.checkbox{min-height:18px;padding-left:18px}.radio input[type="radio"],.checkbox input[type="checkbox"]{float:left;margin-left:-18px}.controls>.radio:first-child,.controls>.checkbox:first-child{padding-top:5px}.radio.inline,.checkbox.inline{display:inline-block;padding-top:5px;margin-bottom:0;vertical-align:middle}.radio.inline+.radio.inline,.checkbox.inline+.checkbox.inline{margin-left:10px}.input-mini{width:60px}.input-small{width:90px}.input-medium{width:150px}.input-large{width:210px}.input-xlarge{width:270px}.input-xxlarge{width:530px}input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input[class*="span"],.row-fluid input[class*="span"],.row-fluid select[class*="span"],.row-fluid textarea[class*="span"],.row-fluid .uneditable-input[class*="span"]{float:none;margin-left:0}.input-append input[class*="span"],.input-append .uneditable-input[class*="span"],.input-prepend input[class*="span"],.input-prepend .uneditable-input[class*="span"],.row-fluid .input-prepend [class*="span"],.row-fluid .input-append [class*="span"]{display:inline-block}input,textarea,.uneditable-input{margin-left:0}input.span12,textarea.span12,.uneditable-input.span12{width:930px}input.span11,textarea.span11,.uneditable-input.span11{width:850px}input.span10,textarea.span10,.uneditable-input.span10{width:770px}input.span9,textarea.span9,.uneditable-input.span9{width:690px}input.span8,textarea.span8,.uneditable-input.span8{width:610px}input.span7,textarea.span7,.uneditable-input.span7{width:530px}input.span6,textarea.span6,.uneditable-input.span6{width:450px}input.span5,textarea.span5,.uneditable-input.span5{width:370px}input.span4,textarea.span4,.uneditable-input.span4{width:290px}input.span3,textarea.span3,.uneditable-input.span3{width:210px}input.span2,textarea.span2,.uneditable-input.span2{width:130px}input.span1,textarea.span1,.uneditable-input.span1{width:50px}input[disabled],select[disabled],textarea[disabled],input[readonly],select[readonly],textarea[readonly]{cursor:not-allowed;background-color:#eee;border-color:#ddd}input[type="radio"][disabled],input[type="checkbox"][disabled],input[type="radio"][readonly],input[type="checkbox"][readonly]{background-color:transparent}.control-group.warning>label,.control-group.warning .help-block,.control-group.warning .help-inline{color:#c09853}.control-group.warning .checkbox,.control-group.warning .radio,.control-group.warning input,.control-group.warning select,.control-group.warning textarea{color:#c09853;border-color:#c09853}.control-group.warning .checkbox:focus,.control-group.warning .radio:focus,.control-group.warning input:focus,.control-group.warning select:focus,.control-group.warning textarea:focus{border-color:#a47e3c;-webkit-box-shadow:0 0 6px #dbc59e;-moz-box-shadow:0 0 6px #dbc59e;box-shadow:0 0 6px #dbc59e}.control-group.warning .input-prepend .add-on,.control-group.warning .input-append .add-on{color:#c09853;background-color:#fcf8e3;border-color:#c09853}.control-group.error>label,.control-group.error .help-block,.control-group.error .help-inline{color:#b94a48}.control-group.error .checkbox,.control-group.error .radio,.control-group.error input,.control-group.error select,.control-group.error textarea{color:#b94a48;border-color:#b94a48}.control-group.error .checkbox:focus,.control-group.error .radio:focus,.control-group.error input:focus,.control-group.error select:focus,.control-group.error textarea:focus{border-color:#953b39;-webkit-box-shadow:0 0 6px #d59392;-moz-box-shadow:0 0 6px #d59392;box-shadow:0 0 6px #d59392}.control-group.error .input-prepend .add-on,.control-group.error .input-append .add-on{color:#b94a48;background-color:#f2dede;border-color:#b94a48}.control-group.success>label,.control-group.success .help-block,.control-group.success .help-inline{color:#468847}.control-group.success .checkbox,.control-group.success .radio,.control-group.success input,.control-group.success select,.control-group.success textarea{color:#468847;border-color:#468847}.control-group.success .checkbox:focus,.control-group.success .radio:focus,.control-group.success input:focus,.control-group.success select:focus,.control-group.success textarea:focus{border-color:#356635;-webkit-box-shadow:0 0 6px #7aba7b;-moz-box-shadow:0 0 6px #7aba7b;box-shadow:0 0 6px #7aba7b}.control-group.success .input-prepend .add-on,.control-group.success .input-append .add-on{color:#468847;background-color:#dff0d8;border-color:#468847}input:focus:required:invalid,textarea:focus:required:invalid,select:focus:required:invalid{color:#b94a48;border-color:#ee5f5b}input:focus:required:invalid:focus,textarea:focus:required:invalid:focus,select:focus:required:invalid:focus{border-color:#e9322d;-webkit-box-shadow:0 0 6px #f8b9b7;-moz-box-shadow:0 0 6px #f8b9b7;box-shadow:0 0 6px #f8b9b7}.form-actions{padding:17px 20px 18px;margin-top:18px;margin-bottom:18px;background-color:#f5f5f5;border-top:1px solid #e5e5e5;*zoom:1}.form-actions:before,.form-actions:after{display:table;content:""}.form-actions:after{clear:both}.uneditable-input{overflow:hidden;white-space:nowrap;cursor:not-allowed;background-color:#fff;border-color:#eee;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.025);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.025);box-shadow:inset 0 1px 2px rgba(0,0,0,0.025)}:-moz-placeholder{color:#999}:-ms-input-placeholder{color:#999}::-webkit-input-placeholder{color:#999}.help-block,.help-inline{color:#555}.help-block{display:block;margin-bottom:9px}.help-inline{display:inline-block;*display:inline;padding-left:5px;vertical-align:middle;*zoom:1}.input-prepend,.input-append{margin-bottom:5px}.input-prepend input,.input-append input,.input-prepend select,.input-append select,.input-prepend .uneditable-input,.input-append .uneditable-input{position:relative;margin-bottom:0;*margin-left:0;vertical-align:middle;-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.input-prepend input:focus,.input-append input:focus,.input-prepend select:focus,.input-append select:focus,.input-prepend .uneditable-input:focus,.input-append .uneditable-input:focus{z-index:2}.input-prepend .uneditable-input,.input-append .uneditable-input{border-left-color:#ccc}.input-prepend .add-on,.input-append .add-on{display:inline-block;width:auto;height:18px;min-width:16px;padding:4px 5px;font-weight:normal;line-height:18px;text-align:center;text-shadow:0 1px 0 #fff;vertical-align:middle;background-color:#eee;border:1px solid #ccc}.input-prepend .add-on,.input-append .add-on,.input-prepend .btn,.input-append .btn{margin-left:-1px;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.input-prepend .active,.input-append .active{background-color:#a9dba9;border-color:#46a546}.input-prepend .add-on,.input-prepend .btn{margin-right:-1px}.input-prepend .add-on:first-child,.input-prepend .btn:first-child{-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-append input,.input-append select,.input-append .uneditable-input{-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-append .uneditable-input{border-right-color:#ccc;border-left-color:#eee}.input-append .add-on:last-child,.input-append .btn:last-child{-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.input-prepend.input-append input,.input-prepend.input-append select,.input-prepend.input-append .uneditable-input{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.input-prepend.input-append .add-on:first-child,.input-prepend.input-append .btn:first-child{margin-right:-1px;-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-prepend.input-append .add-on:last-child,.input-prepend.input-append .btn:last-child{margin-left:-1px;-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.search-query{padding-right:14px;padding-right:4px \9;padding-left:14px;padding-left:4px \9;margin-bottom:0;-webkit-border-radius:14px;-moz-border-radius:14px;border-radius:14px}.form-search input,.form-inline input,.form-horizontal input,.form-search textarea,.form-inline textarea,.form-horizontal textarea,.form-search select,.form-inline select,.form-horizontal select,.form-search .help-inline,.form-inline .help-inline,.form-horizontal .help-inline,.form-search .uneditable-input,.form-inline .uneditable-input,.form-horizontal .uneditable-input,.form-search .input-prepend,.form-inline .input-prepend,.form-horizontal .input-prepend,.form-search .input-append,.form-inline .input-append,.form-horizontal .input-append{display:inline-block;*display:inline;margin-bottom:0;*zoom:1}.form-search .hide,.form-inline .hide,.form-horizontal .hide{display:none}.form-search label,.form-inline label{display:inline-block}.form-search .input-append,.form-inline .input-append,.form-search .input-prepend,.form-inline .input-prepend{margin-bottom:0}.form-search .radio,.form-search .checkbox,.form-inline .radio,.form-inline .checkbox{padding-left:0;margin-bottom:0;vertical-align:middle}.form-search .radio input[type="radio"],.form-search .checkbox input[type="checkbox"],.form-inline .radio input[type="radio"],.form-inline .checkbox input[type="checkbox"]{float:left;margin-right:3px;margin-left:0}.control-group{margin-bottom:9px}legend+.control-group{margin-top:18px;-webkit-margin-top-collapse:separate}.form-horizontal .control-group{margin-bottom:18px;*zoom:1}.form-horizontal .control-group:before,.form-horizontal .control-group:after{display:table;content:""}.form-horizontal .control-group:after{clear:both}.form-horizontal .control-label{float:left;width:140px;padding-top:5px;text-align:right}.form-horizontal .controls{*display:inline-block;*padding-left:20px;margin-left:160px;*margin-left:0}.form-horizontal .controls:first-child{*padding-left:160px}.form-horizontal .help-block{margin-top:9px;margin-bottom:0}.form-horizontal .form-actions{padding-left:160px}table{max-width:100%;background-color:transparent;border-collapse:collapse;border-spacing:0}.table{width:100%;margin-bottom:18px}.table th,.table td{padding:8px;line-height:18px;text-align:left;vertical-align:top;border-top:1px solid #ddd}.table th{font-weight:bold}.table thead th{vertical-align:bottom}.table caption+thead tr:first-child th,.table caption+thead tr:first-child td,.table colgroup+thead tr:first-child th,.table colgroup+thead tr:first-child td,.table thead:first-child tr:first-child th,.table thead:first-child tr:first-child td{border-top:0}.table tbody+tbody{border-top:2px solid #ddd}.table-condensed th,.table-condensed td{padding:4px 5px}.table-bordered{border:1px solid #ddd;border-collapse:separate;*border-collapse:collapsed;border-left:0;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.table-bordered th,.table-bordered td{border-left:1px solid #ddd}.table-bordered caption+thead tr:first-child th,.table-bordered caption+tbody tr:first-child th,.table-bordered caption+tbody tr:first-child td,.table-bordered colgroup+thead tr:first-child th,.table-bordered colgroup+tbody tr:first-child th,.table-bordered colgroup+tbody tr:first-child td,.table-bordered thead:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child td{border-top:0}.table-bordered thead:first-child tr:first-child th:first-child,.table-bordered tbody:first-child tr:first-child td:first-child{-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-topleft:4px}.table-bordered thead:first-child tr:first-child th:last-child,.table-bordered tbody:first-child tr:first-child td:last-child{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-moz-border-radius-topright:4px}.table-bordered thead:last-child tr:last-child th:first-child,.table-bordered tbody:last-child tr:last-child td:first-child{-webkit-border-radius:0 0 0 4px;-moz-border-radius:0 0 0 4px;border-radius:0 0 0 4px;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-moz-border-radius-bottomleft:4px}.table-bordered thead:last-child tr:last-child th:last-child,.table-bordered tbody:last-child tr:last-child td:last-child{-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-moz-border-radius-bottomright:4px}.table-striped tbody tr:nth-child(odd) td,.table-striped tbody tr:nth-child(odd) th{background-color:#f9f9f9}.table tbody tr:hover td,.table tbody tr:hover th{background-color:#f5f5f5}table .span1{float:none;width:44px;margin-left:0}table .span2{float:none;width:124px;margin-left:0}table .span3{float:none;width:204px;margin-left:0}table .span4{float:none;width:284px;margin-left:0}table .span5{float:none;width:364px;margin-left:0}table .span6{float:none;width:444px;margin-left:0}table .span7{float:none;width:524px;margin-left:0}table .span8{float:none;width:604px;margin-left:0}table .span9{float:none;width:684px;margin-left:0}table .span10{float:none;width:764px;margin-left:0}table .span11{float:none;width:844px;margin-left:0}table .span12{float:none;width:924px;margin-left:0}table .span13{float:none;width:1004px;margin-left:0}table .span14{float:none;width:1084px;margin-left:0}table .span15{float:none;width:1164px;margin-left:0}table .span16{float:none;width:1244px;margin-left:0}table .span17{float:none;width:1324px;margin-left:0}table .span18{float:none;width:1404px;margin-left:0}table .span19{float:none;width:1484px;margin-left:0}table .span20{float:none;width:1564px;margin-left:0}table .span21{float:none;width:1644px;margin-left:0}table .span22{float:none;width:1724px;margin-left:0}table .span23{float:none;width:1804px;margin-left:0}table .span24{float:none;width:1884px;margin-left:0}[class^="icon-"],[class*=" icon-"]{display:inline-block;width:14px;height:14px;*margin-right:.3em;line-height:14px;vertical-align:text-top;background-image:url("../img/glyphicons-halflings.png");background-position:14px 14px;background-repeat:no-repeat}[class^="icon-"]:last-child,[class*=" icon-"]:last-child{*margin-left:0}.icon-white{background-image:url("../img/glyphicons-halflings-white.png")}.icon-glass{background-position:0 0}.icon-music{background-position:-24px 0}.icon-search{background-position:-48px 0}.icon-envelope{background-position:-72px 0}.icon-heart{background-position:-96px 0}.icon-star{background-position:-120px 0}.icon-star-empty{background-position:-144px 0}.icon-user{background-position:-168px 0}.icon-film{background-position:-192px 0}.icon-th-large{background-position:-216px 0}.icon-th{background-position:-240px 0}.icon-th-list{background-position:-264px 0}.icon-ok{background-position:-288px 0}.icon-remove{background-position:-312px 0}.icon-zoom-in{background-position:-336px 0}.icon-zoom-out{background-position:-360px 0}.icon-off{background-position:-384px 0}.icon-signal{background-position:-408px 0}.icon-cog{background-position:-432px 0}.icon-trash{background-position:-456px 0}.icon-home{background-position:0 -24px}.icon-file{background-position:-24px -24px}.icon-time{background-position:-48px -24px}.icon-road{background-position:-72px -24px}.icon-download-alt{background-position:-96px -24px}.icon-download{background-position:-120px -24px}.icon-upload{background-position:-144px -24px}.icon-inbox{background-position:-168px -24px}.icon-play-circle{background-position:-192px -24px}.icon-repeat{background-position:-216px -24px}.icon-refresh{background-position:-240px -24px}.icon-list-alt{background-position:-264px -24px}.icon-lock{background-position:-287px -24px}.icon-flag{background-position:-312px -24px}.icon-headphones{background-position:-336px -24px}.icon-volume-off{background-position:-360px -24px}.icon-volume-down{background-position:-384px -24px}.icon-volume-up{background-position:-408px -24px}.icon-qrcode{background-position:-432px -24px}.icon-barcode{background-position:-456px -24px}.icon-tag{background-position:0 -48px}.icon-tags{background-position:-25px -48px}.icon-book{background-position:-48px -48px}.icon-bookmark{background-position:-72px -48px}.icon-print{background-position:-96px -48px}.icon-camera{background-position:-120px -48px}.icon-font{background-position:-144px -48px}.icon-bold{background-position:-167px -48px}.icon-italic{background-position:-192px -48px}.icon-text-height{background-position:-216px -48px}.icon-text-width{background-position:-240px -48px}.icon-align-left{background-position:-264px -48px}.icon-align-center{background-position:-288px -48px}.icon-align-right{background-position:-312px -48px}.icon-align-justify{background-position:-336px -48px}.icon-list{background-position:-360px -48px}.icon-indent-left{background-position:-384px -48px}.icon-indent-right{background-position:-408px -48px}.icon-facetime-video{background-position:-432px -48px}.icon-picture{background-position:-456px -48px}.icon-pencil{background-position:0 -72px}.icon-map-marker{background-position:-24px -72px}.icon-adjust{background-position:-48px -72px}.icon-tint{background-position:-72px -72px}.icon-edit{background-position:-96px -72px}.icon-share{background-position:-120px -72px}.icon-check{background-position:-144px -72px}.icon-move{background-position:-168px -72px}.icon-step-backward{background-position:-192px -72px}.icon-fast-backward{background-position:-216px -72px}.icon-backward{background-position:-240px -72px}.icon-play{background-position:-264px -72px}.icon-pause{background-position:-288px -72px}.icon-stop{background-position:-312px -72px}.icon-forward{background-position:-336px -72px}.icon-fast-forward{background-position:-360px -72px}.icon-step-forward{background-position:-384px -72px}.icon-eject{background-position:-408px -72px}.icon-chevron-left{background-position:-432px -72px}.icon-chevron-right{background-position:-456px -72px}.icon-plus-sign{background-position:0 -96px}.icon-minus-sign{background-position:-24px -96px}.icon-remove-sign{background-position:-48px -96px}.icon-ok-sign{background-position:-72px -96px}.icon-question-sign{background-position:-96px -96px}.icon-info-sign{background-position:-120px -96px}.icon-screenshot{background-position:-144px -96px}.icon-remove-circle{background-position:-168px -96px}.icon-ok-circle{background-position:-192px -96px}.icon-ban-circle{background-position:-216px -96px}.icon-arrow-left{background-position:-240px -96px}.icon-arrow-right{background-position:-264px -96px}.icon-arrow-up{background-position:-289px -96px}.icon-arrow-down{background-position:-312px -96px}.icon-share-alt{background-position:-336px -96px}.icon-resize-full{background-position:-360px -96px}.icon-resize-small{background-position:-384px -96px}.icon-plus{background-position:-408px -96px}.icon-minus{background-position:-433px -96px}.icon-asterisk{background-position:-456px -96px}.icon-exclamation-sign{background-position:0 -120px}.icon-gift{background-position:-24px -120px}.icon-leaf{background-position:-48px -120px}.icon-fire{background-position:-72px -120px}.icon-eye-open{background-position:-96px -120px}.icon-eye-close{background-position:-120px -120px}.icon-warning-sign{background-position:-144px -120px}.icon-plane{background-position:-168px -120px}.icon-calendar{background-position:-192px -120px}.icon-random{background-position:-216px -120px}.icon-comment{background-position:-240px -120px}.icon-magnet{background-position:-264px -120px}.icon-chevron-up{background-position:-288px -120px}.icon-chevron-down{background-position:-313px -119px}.icon-retweet{background-position:-336px -120px}.icon-shopping-cart{background-position:-360px -120px}.icon-folder-close{background-position:-384px -120px}.icon-folder-open{background-position:-408px -120px}.icon-resize-vertical{background-position:-432px -119px}.icon-resize-horizontal{background-position:-456px -118px}.icon-hdd{background-position:0 -144px}.icon-bullhorn{background-position:-24px -144px}.icon-bell{background-position:-48px -144px}.icon-certificate{background-position:-72px -144px}.icon-thumbs-up{background-position:-96px -144px}.icon-thumbs-down{background-position:-120px -144px}.icon-hand-right{background-position:-144px -144px}.icon-hand-left{background-position:-168px -144px}.icon-hand-up{background-position:-192px -144px}.icon-hand-down{background-position:-216px -144px}.icon-circle-arrow-right{background-position:-240px -144px}.icon-circle-arrow-left{background-position:-264px -144px}.icon-circle-arrow-up{background-position:-288px -144px}.icon-circle-arrow-down{background-position:-312px -144px}.icon-globe{background-position:-336px -144px}.icon-wrench{background-position:-360px -144px}.icon-tasks{background-position:-384px -144px}.icon-filter{background-position:-408px -144px}.icon-briefcase{background-position:-432px -144px}.icon-fullscreen{background-position:-456px -144px}.dropup,.dropdown{position:relative}.dropdown-toggle{*margin-bottom:-3px}.dropdown-toggle:active,.open .dropdown-toggle{outline:0}.caret{display:inline-block;width:0;height:0;vertical-align:top;border-top:4px solid #000;border-right:4px solid transparent;border-left:4px solid transparent;content:"";opacity:.3;filter:alpha(opacity=30)}.dropdown .caret{margin-top:8px;margin-left:2px}.dropdown:hover .caret,.open .caret{opacity:1;filter:alpha(opacity=100)}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:4px 0;margin:1px 0 0;list-style:none;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.2);*border-right-width:2px;*border-bottom-width:2px;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px;-webkit-box-shadow:0 5px 10px rgba(0,0,0,0.2);-moz-box-shadow:0 5px 10px rgba(0,0,0,0.2);box-shadow:0 5px 10px rgba(0,0,0,0.2);-webkit-background-clip:padding-box;-moz-background-clip:padding;background-clip:padding-box}.dropdown-menu.pull-right{right:0;left:auto}.dropdown-menu .divider{*width:100%;height:1px;margin:8px 1px;*margin:-5px 0 5px;overflow:hidden;background-color:#e5e5e5;border-bottom:1px solid #fff}.dropdown-menu a{display:block;padding:3px 15px;clear:both;font-weight:normal;line-height:18px;color:#333;white-space:nowrap}.dropdown-menu li>a:hover,.dropdown-menu .active>a,.dropdown-menu .active>a:hover{color:#fff;text-decoration:none;background-color:#08c}.open{*z-index:1000}.open>.dropdown-menu{display:block}.pull-right>.dropdown-menu{right:0;left:auto}.dropup .caret,.navbar-fixed-bottom .dropdown .caret{border-top:0;border-bottom:4px solid #000;content:"\2191"}.dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:1px}.typeahead{margin-top:2px;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.well{min-height:20px;padding:19px;margin-bottom:20px;background-color:#f5f5f5;border:1px solid #eee;border:1px solid rgba(0,0,0,0.05);-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.05);box-shadow:inset 0 1px 1px rgba(0,0,0,0.05)}.well blockquote{border-color:#ddd;border-color:rgba(0,0,0,0.15)}.well-large{padding:24px;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.well-small{padding:9px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.fade{opacity:0;-webkit-transition:opacity .15s linear;-moz-transition:opacity .15s linear;-ms-transition:opacity .15s linear;-o-transition:opacity .15s linear;transition:opacity .15s linear}.fade.in{opacity:1}.collapse{position:relative;height:0;overflow:hidden;-webkit-transition:height .35s ease;-moz-transition:height .35s ease;-ms-transition:height .35s ease;-o-transition:height .35s ease;transition:height .35s ease}.collapse.in{height:auto}.close{float:right;font-size:20px;font-weight:bold;line-height:18px;color:#000;text-shadow:0 1px 0 #fff;opacity:.2;filter:alpha(opacity=20)}.close:hover{color:#000;text-decoration:none;cursor:pointer;opacity:.4;filter:alpha(opacity=40)}button.close{padding:0;cursor:pointer;background:transparent;border:0;-webkit-appearance:none}.btn{display:inline-block;*display:inline;padding:4px 10px 4px;margin-bottom:0;*margin-left:.3em;font-size:13px;line-height:18px;*line-height:20px;color:#333;text-align:center;text-shadow:0 1px 1px rgba(255,255,255,0.75);vertical-align:middle;cursor:pointer;background-color:#f5f5f5;*background-color:#e6e6e6;background-image:-ms-linear-gradient(top,#fff,#e6e6e6);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fff),to(#e6e6e6));background-image:-webkit-linear-gradient(top,#fff,#e6e6e6);background-image:-o-linear-gradient(top,#fff,#e6e6e6);background-image:linear-gradient(top,#fff,#e6e6e6);background-image:-moz-linear-gradient(top,#fff,#e6e6e6);background-repeat:repeat-x;border:1px solid #ccc;*border:0;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);border-color:#e6e6e6 #e6e6e6 #bfbfbf;border-bottom-color:#b3b3b3;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffffff',endColorstr='#e6e6e6',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false);*zoom:1;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05)}.btn:hover,.btn:active,.btn.active,.btn.disabled,.btn[disabled]{background-color:#e6e6e6;*background-color:#d9d9d9}.btn:active,.btn.active{background-color:#ccc \9}.btn:first-child{*margin-left:0}.btn:hover{color:#333;text-decoration:none;background-color:#e6e6e6;*background-color:#d9d9d9;background-position:0 -15px;-webkit-transition:background-position .1s linear;-moz-transition:background-position .1s linear;-ms-transition:background-position .1s linear;-o-transition:background-position .1s linear;transition:background-position .1s linear}.btn:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.btn.active,.btn:active{background-color:#e6e6e6;background-color:#d9d9d9 \9;background-image:none;outline:0;-webkit-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05)}.btn.disabled,.btn[disabled]{cursor:default;background-color:#e6e6e6;background-image:none;opacity:.65;filter:alpha(opacity=65);-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.btn-large{padding:9px 14px;font-size:15px;line-height:normal;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px}.btn-large [class^="icon-"]{margin-top:1px}.btn-small{padding:5px 9px;font-size:11px;line-height:16px}.btn-small [class^="icon-"]{margin-top:-1px}.btn-mini{padding:2px 6px;font-size:11px;line-height:14px}.btn-primary,.btn-primary:hover,.btn-warning,.btn-warning:hover,.btn-danger,.btn-danger:hover,.btn-success,.btn-success:hover,.btn-info,.btn-info:hover,.btn-inverse,.btn-inverse:hover{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25)}.btn-primary.active,.btn-warning.active,.btn-danger.active,.btn-success.active,.btn-info.active,.btn-inverse.active{color:rgba(255,255,255,0.75)}.btn{border-color:#ccc;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25)}.btn-primary{background-color:#0074cc;*background-color:#05c;background-image:-ms-linear-gradient(top,#08c,#05c);background-image:-webkit-gradient(linear,0 0,0 100%,from(#08c),to(#05c));background-image:-webkit-linear-gradient(top,#08c,#05c);background-image:-o-linear-gradient(top,#08c,#05c);background-image:-moz-linear-gradient(top,#08c,#05c);background-image:linear-gradient(top,#08c,#05c);background-repeat:repeat-x;border-color:#05c #05c #003580;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#0088cc',endColorstr='#0055cc',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-primary:hover,.btn-primary:active,.btn-primary.active,.btn-primary.disabled,.btn-primary[disabled]{background-color:#05c;*background-color:#004ab3}.btn-primary:active,.btn-primary.active{background-color:#004099 \9}.btn-warning{background-color:#faa732;*background-color:#f89406;background-image:-ms-linear-gradient(top,#fbb450,#f89406);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fbb450),to(#f89406));background-image:-webkit-linear-gradient(top,#fbb450,#f89406);background-image:-o-linear-gradient(top,#fbb450,#f89406);background-image:-moz-linear-gradient(top,#fbb450,#f89406);background-image:linear-gradient(top,#fbb450,#f89406);background-repeat:repeat-x;border-color:#f89406 #f89406 #ad6704;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fbb450',endColorstr='#f89406',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-warning:hover,.btn-warning:active,.btn-warning.active,.btn-warning.disabled,.btn-warning[disabled]{background-color:#f89406;*background-color:#df8505}.btn-warning:active,.btn-warning.active{background-color:#c67605 \9}.btn-danger{background-color:#da4f49;*background-color:#bd362f;background-image:-ms-linear-gradient(top,#ee5f5b,#bd362f);background-image:-webkit-gradient(linear,0 0,0 100%,from(#ee5f5b),to(#bd362f));background-image:-webkit-linear-gradient(top,#ee5f5b,#bd362f);background-image:-o-linear-gradient(top,#ee5f5b,#bd362f);background-image:-moz-linear-gradient(top,#ee5f5b,#bd362f);background-image:linear-gradient(top,#ee5f5b,#bd362f);background-repeat:repeat-x;border-color:#bd362f #bd362f #802420;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ee5f5b',endColorstr='#bd362f',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-danger:hover,.btn-danger:active,.btn-danger.active,.btn-danger.disabled,.btn-danger[disabled]{background-color:#bd362f;*background-color:#a9302a}.btn-danger:active,.btn-danger.active{background-color:#942a25 \9}.btn-success{background-color:#5bb75b;*background-color:#51a351;background-image:-ms-linear-gradient(top,#62c462,#51a351);background-image:-webkit-gradient(linear,0 0,0 100%,from(#62c462),to(#51a351));background-image:-webkit-linear-gradient(top,#62c462,#51a351);background-image:-o-linear-gradient(top,#62c462,#51a351);background-image:-moz-linear-gradient(top,#62c462,#51a351);background-image:linear-gradient(top,#62c462,#51a351);background-repeat:repeat-x;border-color:#51a351 #51a351 #387038;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#62c462',endColorstr='#51a351',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-success:hover,.btn-success:active,.btn-success.active,.btn-success.disabled,.btn-success[disabled]{background-color:#51a351;*background-color:#499249}.btn-success:active,.btn-success.active{background-color:#408140 \9}.btn-info{background-color:#49afcd;*background-color:#2f96b4;background-image:-ms-linear-gradient(top,#5bc0de,#2f96b4);background-image:-webkit-gradient(linear,0 0,0 100%,from(#5bc0de),to(#2f96b4));background-image:-webkit-linear-gradient(top,#5bc0de,#2f96b4);background-image:-o-linear-gradient(top,#5bc0de,#2f96b4);background-image:-moz-linear-gradient(top,#5bc0de,#2f96b4);background-image:linear-gradient(top,#5bc0de,#2f96b4);background-repeat:repeat-x;border-color:#2f96b4 #2f96b4 #1f6377;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#5bc0de',endColorstr='#2f96b4',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-info:hover,.btn-info:active,.btn-info.active,.btn-info.disabled,.btn-info[disabled]{background-color:#2f96b4;*background-color:#2a85a0}.btn-info:active,.btn-info.active{background-color:#24748c \9}.btn-inverse{background-color:#414141;*background-color:#222;background-image:-ms-linear-gradient(top,#555,#222);background-image:-webkit-gradient(linear,0 0,0 100%,from(#555),to(#222));background-image:-webkit-linear-gradient(top,#555,#222);background-image:-o-linear-gradient(top,#555,#222);background-image:-moz-linear-gradient(top,#555,#222);background-image:linear-gradient(top,#555,#222);background-repeat:repeat-x;border-color:#222 #222 #000;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#555555',endColorstr='#222222',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-inverse:hover,.btn-inverse:active,.btn-inverse.active,.btn-inverse.disabled,.btn-inverse[disabled]{background-color:#222;*background-color:#151515}.btn-inverse:active,.btn-inverse.active{background-color:#080808 \9}button.btn,input[type="submit"].btn{*padding-top:2px;*padding-bottom:2px}button.btn::-moz-focus-inner,input[type="submit"].btn::-moz-focus-inner{padding:0;border:0}button.btn.btn-large,input[type="submit"].btn.btn-large{*padding-top:7px;*padding-bottom:7px}button.btn.btn-small,input[type="submit"].btn.btn-small{*padding-top:3px;*padding-bottom:3px}button.btn.btn-mini,input[type="submit"].btn.btn-mini{*padding-top:1px;*padding-bottom:1px}.btn-group{position:relative;*margin-left:.3em;*zoom:1}.btn-group:before,.btn-group:after{display:table;content:""}.btn-group:after{clear:both}.btn-group:first-child{*margin-left:0}.btn-group+.btn-group{margin-left:5px}.btn-toolbar{margin-top:9px;margin-bottom:9px}.btn-toolbar .btn-group{display:inline-block;*display:inline;*zoom:1}.btn-group>.btn{position:relative;float:left;margin-left:-1px;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.btn-group>.btn:first-child{margin-left:0;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-bottomleft:4px;-moz-border-radius-topleft:4px}.btn-group>.btn:last-child,.btn-group>.dropdown-toggle{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-moz-border-radius-topright:4px;-moz-border-radius-bottomright:4px}.btn-group>.btn.large:first-child{margin-left:0;-webkit-border-bottom-left-radius:6px;border-bottom-left-radius:6px;-webkit-border-top-left-radius:6px;border-top-left-radius:6px;-moz-border-radius-bottomleft:6px;-moz-border-radius-topleft:6px}.btn-group>.btn.large:last-child,.btn-group>.large.dropdown-toggle{-webkit-border-top-right-radius:6px;border-top-right-radius:6px;-webkit-border-bottom-right-radius:6px;border-bottom-right-radius:6px;-moz-border-radius-topright:6px;-moz-border-radius-bottomright:6px}.btn-group>.btn:hover,.btn-group>.btn:focus,.btn-group>.btn:active,.btn-group>.btn.active{z-index:2}.btn-group .dropdown-toggle:active,.btn-group.open .dropdown-toggle{outline:0}.btn-group>.dropdown-toggle{*padding-top:4px;padding-right:8px;*padding-bottom:4px;padding-left:8px;-webkit-box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05)}.btn-group>.btn-mini.dropdown-toggle{padding-right:5px;padding-left:5px}.btn-group>.btn-small.dropdown-toggle{*padding-top:4px;*padding-bottom:4px}.btn-group>.btn-large.dropdown-toggle{padding-right:12px;padding-left:12px}.btn-group.open .dropdown-toggle{background-image:none;-webkit-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05)}.btn-group.open .btn.dropdown-toggle{background-color:#e6e6e6}.btn-group.open .btn-primary.dropdown-toggle{background-color:#05c}.btn-group.open .btn-warning.dropdown-toggle{background-color:#f89406}.btn-group.open .btn-danger.dropdown-toggle{background-color:#bd362f}.btn-group.open .btn-success.dropdown-toggle{background-color:#51a351}.btn-group.open .btn-info.dropdown-toggle{background-color:#2f96b4}.btn-group.open .btn-inverse.dropdown-toggle{background-color:#222}.btn .caret{margin-top:7px;margin-left:0}.btn:hover .caret,.open.btn-group .caret{opacity:1;filter:alpha(opacity=100)}.btn-mini .caret{margin-top:5px}.btn-small .caret{margin-top:6px}.btn-large .caret{margin-top:6px;border-top-width:5px;border-right-width:5px;border-left-width:5px}.dropup .btn-large .caret{border-top:0;border-bottom:5px solid #000}.btn-primary .caret,.btn-warning .caret,.btn-danger .caret,.btn-info .caret,.btn-success .caret,.btn-inverse .caret{border-top-color:#fff;border-bottom-color:#fff;opacity:.75;filter:alpha(opacity=75)}.alert{padding:8px 35px 8px 14px;margin-bottom:18px;color:#c09853;text-shadow:0 1px 0 rgba(255,255,255,0.5);background-color:#fcf8e3;border:1px solid #fbeed5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.alert-heading{color:inherit}.alert .close{position:relative;top:-2px;right:-21px;line-height:18px}.alert-success{color:#468847;background-color:#dff0d8;border-color:#d6e9c6}.alert-danger,.alert-error{color:#b94a48;background-color:#f2dede;border-color:#eed3d7}.alert-info{color:#3a87ad;background-color:#d9edf7;border-color:#bce8f1}.alert-block{padding-top:14px;padding-bottom:14px}.alert-block>p,.alert-block>ul{margin-bottom:0}.alert-block p+p{margin-top:5px}.nav{margin-bottom:18px;margin-left:0;list-style:none}.nav>li>a{display:block}.nav>li>a:hover{text-decoration:none;background-color:#eee}.nav>.pull-right{float:right}.nav .nav-header{display:block;padding:3px 15px;font-size:11px;font-weight:bold;line-height:18px;color:#999;text-shadow:0 1px 0 rgba(255,255,255,0.5);text-transform:uppercase}.nav li+.nav-header{margin-top:9px}.nav-list{padding-right:15px;padding-left:15px;margin-bottom:0}.nav-list>li>a,.nav-list .nav-header{margin-right:-15px;margin-left:-15px;text-shadow:0 1px 0 rgba(255,255,255,0.5)}.nav-list>li>a{padding:3px 15px}.nav-list>.active>a,.nav-list>.active>a:hover{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.2);background-color:#08c}.nav-list [class^="icon-"]{margin-right:2px}.nav-list .divider{*width:100%;height:1px;margin:8px 1px;*margin:-5px 0 5px;overflow:hidden;background-color:#e5e5e5;border-bottom:1px solid #fff}.nav-tabs,.nav-pills{*zoom:1}.nav-tabs:before,.nav-pills:before,.nav-tabs:after,.nav-pills:after{display:table;content:""}.nav-tabs:after,.nav-pills:after{clear:both}.nav-tabs>li,.nav-pills>li{float:left}.nav-tabs>li>a,.nav-pills>li>a{padding-right:12px;padding-left:12px;margin-right:2px;line-height:14px}.nav-tabs{border-bottom:1px solid #ddd}.nav-tabs>li{margin-bottom:-1px}.nav-tabs>li>a{padding-top:8px;padding-bottom:8px;line-height:18px;border:1px solid transparent;-webkit-border-radius:4px 4px 0 0;-moz-border-radius:4px 4px 0 0;border-radius:4px 4px 0 0}.nav-tabs>li>a:hover{border-color:#eee #eee #ddd}.nav-tabs>.active>a,.nav-tabs>.active>a:hover{color:#555;cursor:default;background-color:#fff;border:1px solid #ddd;border-bottom-color:transparent}.nav-pills>li>a{padding-top:8px;padding-bottom:8px;margin-top:2px;margin-bottom:2px;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px}.nav-pills>.active>a,.nav-pills>.active>a:hover{color:#fff;background-color:#08c}.nav-stacked>li{float:none}.nav-stacked>li>a{margin-right:0}.nav-tabs.nav-stacked{border-bottom:0}.nav-tabs.nav-stacked>li>a{border:1px solid #ddd;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.nav-tabs.nav-stacked>li:first-child>a{-webkit-border-radius:4px 4px 0 0;-moz-border-radius:4px 4px 0 0;border-radius:4px 4px 0 0}.nav-tabs.nav-stacked>li:last-child>a{-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px}.nav-tabs.nav-stacked>li>a:hover{z-index:2;border-color:#ddd}.nav-pills.nav-stacked>li>a{margin-bottom:3px}.nav-pills.nav-stacked>li:last-child>a{margin-bottom:1px}.nav-tabs .dropdown-menu{-webkit-border-radius:0 0 5px 5px;-moz-border-radius:0 0 5px 5px;border-radius:0 0 5px 5px}.nav-pills .dropdown-menu{-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.nav-tabs .dropdown-toggle .caret,.nav-pills .dropdown-toggle .caret{margin-top:6px;border-top-color:#08c;border-bottom-color:#08c}.nav-tabs .dropdown-toggle:hover .caret,.nav-pills .dropdown-toggle:hover .caret{border-top-color:#005580;border-bottom-color:#005580}.nav-tabs .active .dropdown-toggle .caret,.nav-pills .active .dropdown-toggle .caret{border-top-color:#333;border-bottom-color:#333}.nav>.dropdown.active>a:hover{color:#000;cursor:pointer}.nav-tabs .open .dropdown-toggle,.nav-pills .open .dropdown-toggle,.nav>li.dropdown.open.active>a:hover{color:#fff;background-color:#999;border-color:#999}.nav li.dropdown.open .caret,.nav li.dropdown.open.active .caret,.nav li.dropdown.open a:hover .caret{border-top-color:#fff;border-bottom-color:#fff;opacity:1;filter:alpha(opacity=100)}.tabs-stacked .open>a:hover{border-color:#999}.tabbable{*zoom:1}.tabbable:before,.tabbable:after{display:table;content:""}.tabbable:after{clear:both}.tab-content{overflow:auto}.tabs-below>.nav-tabs,.tabs-right>.nav-tabs,.tabs-left>.nav-tabs{border-bottom:0}.tab-content>.tab-pane,.pill-content>.pill-pane{display:none}.tab-content>.active,.pill-content>.active{display:block}.tabs-below>.nav-tabs{border-top:1px solid #ddd}.tabs-below>.nav-tabs>li{margin-top:-1px;margin-bottom:0}.tabs-below>.nav-tabs>li>a{-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px}.tabs-below>.nav-tabs>li>a:hover{border-top-color:#ddd;border-bottom-color:transparent}.tabs-below>.nav-tabs>.active>a,.tabs-below>.nav-tabs>.active>a:hover{border-color:transparent #ddd #ddd #ddd}.tabs-left>.nav-tabs>li,.tabs-right>.nav-tabs>li{float:none}.tabs-left>.nav-tabs>li>a,.tabs-right>.nav-tabs>li>a{min-width:74px;margin-right:0;margin-bottom:3px}.tabs-left>.nav-tabs{float:left;margin-right:19px;border-right:1px solid #ddd}.tabs-left>.nav-tabs>li>a{margin-right:-1px;-webkit-border-radius:4px 0 0 4px;-moz-border-radius:4px 0 0 4px;border-radius:4px 0 0 4px}.tabs-left>.nav-tabs>li>a:hover{border-color:#eee #ddd #eee #eee}.tabs-left>.nav-tabs .active>a,.tabs-left>.nav-tabs .active>a:hover{border-color:#ddd transparent #ddd #ddd;*border-right-color:#fff}.tabs-right>.nav-tabs{float:right;margin-left:19px;border-left:1px solid #ddd}.tabs-right>.nav-tabs>li>a{margin-left:-1px;-webkit-border-radius:0 4px 4px 0;-moz-border-radius:0 4px 4px 0;border-radius:0 4px 4px 0}.tabs-right>.nav-tabs>li>a:hover{border-color:#eee #eee #eee #ddd}.tabs-right>.nav-tabs .active>a,.tabs-right>.nav-tabs .active>a:hover{border-color:#ddd #ddd #ddd transparent;*border-left-color:#fff}.navbar{*position:relative;*z-index:2;margin-bottom:18px;overflow:visible}.navbar-inner{min-height:40px;padding-right:20px;padding-left:20px;background-color:#2c2c2c;background-image:-moz-linear-gradient(top,#333,#222);background-image:-ms-linear-gradient(top,#333,#222);background-image:-webkit-gradient(linear,0 0,0 100%,from(#333),to(#222));background-image:-webkit-linear-gradient(top,#333,#222);background-image:-o-linear-gradient(top,#333,#222);background-image:linear-gradient(top,#333,#222);background-repeat:repeat-x;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#333333',endColorstr='#222222',GradientType=0);-webkit-box-shadow:0 1px 3px rgba(0,0,0,0.25),inset 0 -1px 0 rgba(0,0,0,0.1);-moz-box-shadow:0 1px 3px rgba(0,0,0,0.25),inset 0 -1px 0 rgba(0,0,0,0.1);box-shadow:0 1px 3px rgba(0,0,0,0.25),inset 0 -1px 0 rgba(0,0,0,0.1)}.navbar .container{width:auto}.nav-collapse.collapse{height:auto}.navbar{color:#999}.navbar .brand:hover{text-decoration:none}.navbar .brand{display:block;float:left;padding:8px 20px 12px;margin-left:-20px;font-size:20px;font-weight:200;line-height:1;color:#999}.navbar .navbar-text{margin-bottom:0;line-height:40px}.navbar .navbar-link{color:#999}.navbar .navbar-link:hover{color:#fff}.navbar .btn,.navbar .btn-group{margin-top:5px}.navbar .btn-group .btn{margin:0}.navbar-form{margin-bottom:0;*zoom:1}.navbar-form:before,.navbar-form:after{display:table;content:""}.navbar-form:after{clear:both}.navbar-form input,.navbar-form select,.navbar-form .radio,.navbar-form .checkbox{margin-top:5px}.navbar-form input,.navbar-form select{display:inline-block;margin-bottom:0}.navbar-form input[type="image"],.navbar-form input[type="checkbox"],.navbar-form input[type="radio"]{margin-top:3px}.navbar-form .input-append,.navbar-form .input-prepend{margin-top:6px;white-space:nowrap}.navbar-form .input-append input,.navbar-form .input-prepend input{margin-top:0}.navbar-search{position:relative;float:left;margin-top:6px;margin-bottom:0}.navbar-search .search-query{padding:4px 9px;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:13px;font-weight:normal;line-height:1;color:#fff;background-color:#626262;border:1px solid #151515;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);-webkit-transition:none;-moz-transition:none;-ms-transition:none;-o-transition:none;transition:none}.navbar-search .search-query:-moz-placeholder{color:#ccc}.navbar-search .search-query:-ms-input-placeholder{color:#ccc}.navbar-search .search-query::-webkit-input-placeholder{color:#ccc}.navbar-search .search-query:focus,.navbar-search .search-query.focused{padding:5px 10px;color:#333;text-shadow:0 1px 0 #fff;background-color:#fff;border:0;outline:0;-webkit-box-shadow:0 0 3px rgba(0,0,0,0.15);-moz-box-shadow:0 0 3px rgba(0,0,0,0.15);box-shadow:0 0 3px rgba(0,0,0,0.15)}.navbar-fixed-top,.navbar-fixed-bottom{position:fixed;right:0;left:0;z-index:1030;margin-bottom:0}.navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner{padding-right:0;padding-left:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:940px}.navbar-fixed-top{top:0}.navbar-fixed-bottom{bottom:0}.navbar .nav{position:relative;left:0;display:block;float:left;margin:0 10px 0 0}.navbar .nav.pull-right{float:right}.navbar .nav>li{display:block;float:left}.navbar .nav>li>a{float:none;padding:9px 10px 11px;line-height:19px;color:#999;text-decoration:none;text-shadow:0 -1px 0 rgba(0,0,0,0.25)}.navbar .btn{display:inline-block;padding:4px 10px 4px;margin:5px 5px 6px;line-height:18px}.navbar .btn-group{padding:5px 5px 6px;margin:0}.navbar .nav>li>a:hover{color:#fff;text-decoration:none;background-color:transparent}.navbar .nav .active>a,.navbar .nav .active>a:hover{color:#fff;text-decoration:none;background-color:#222}.navbar .divider-vertical{width:1px;height:40px;margin:0 9px;overflow:hidden;background-color:#222;border-right:1px solid #333}.navbar .nav.pull-right{margin-right:0;margin-left:10px}.navbar .btn-navbar{display:none;float:right;padding:7px 10px;margin-right:5px;margin-left:5px;background-color:#2c2c2c;*background-color:#222;background-image:-ms-linear-gradient(top,#333,#222);background-image:-webkit-gradient(linear,0 0,0 100%,from(#333),to(#222));background-image:-webkit-linear-gradient(top,#333,#222);background-image:-o-linear-gradient(top,#333,#222);background-image:linear-gradient(top,#333,#222);background-image:-moz-linear-gradient(top,#333,#222);background-repeat:repeat-x;border-color:#222 #222 #000;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#333333',endColorstr='#222222',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false);-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075)}.navbar .btn-navbar:hover,.navbar .btn-navbar:active,.navbar .btn-navbar.active,.navbar .btn-navbar.disabled,.navbar .btn-navbar[disabled]{background-color:#222;*background-color:#151515}.navbar .btn-navbar:active,.navbar .btn-navbar.active{background-color:#080808 \9}.navbar .btn-navbar .icon-bar{display:block;width:18px;height:2px;background-color:#f5f5f5;-webkit-border-radius:1px;-moz-border-radius:1px;border-radius:1px;-webkit-box-shadow:0 1px 0 rgba(0,0,0,0.25);-moz-box-shadow:0 1px 0 rgba(0,0,0,0.25);box-shadow:0 1px 0 rgba(0,0,0,0.25)}.btn-navbar .icon-bar+.icon-bar{margin-top:3px}.navbar .dropdown-menu:before{position:absolute;top:-7px;left:9px;display:inline-block;border-right:7px solid transparent;border-bottom:7px solid #ccc;border-left:7px solid transparent;border-bottom-color:rgba(0,0,0,0.2);content:''}.navbar .dropdown-menu:after{position:absolute;top:-6px;left:10px;display:inline-block;border-right:6px solid transparent;border-bottom:6px solid #fff;border-left:6px solid transparent;content:''}.navbar-fixed-bottom .dropdown-menu:before{top:auto;bottom:-7px;border-top:7px solid #ccc;border-bottom:0;border-top-color:rgba(0,0,0,0.2)}.navbar-fixed-bottom .dropdown-menu:after{top:auto;bottom:-6px;border-top:6px solid #fff;border-bottom:0}.navbar .nav li.dropdown .dropdown-toggle .caret,.navbar .nav li.dropdown.open .caret{border-top-color:#fff;border-bottom-color:#fff}.navbar .nav li.dropdown.active .caret{opacity:1;filter:alpha(opacity=100)}.navbar .nav li.dropdown.open>.dropdown-toggle,.navbar .nav li.dropdown.active>.dropdown-toggle,.navbar .nav li.dropdown.open.active>.dropdown-toggle{background-color:transparent}.navbar .nav li.dropdown.active>.dropdown-toggle:hover{color:#fff}.navbar .pull-right .dropdown-menu,.navbar .dropdown-menu.pull-right{right:0;left:auto}.navbar .pull-right .dropdown-menu:before,.navbar .dropdown-menu.pull-right:before{right:12px;left:auto}.navbar .pull-right .dropdown-menu:after,.navbar .dropdown-menu.pull-right:after{right:13px;left:auto}.breadcrumb{padding:7px 14px;margin:0 0 18px;list-style:none;background-color:#fbfbfb;background-image:-moz-linear-gradient(top,#fff,#f5f5f5);background-image:-ms-linear-gradient(top,#fff,#f5f5f5);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fff),to(#f5f5f5));background-image:-webkit-linear-gradient(top,#fff,#f5f5f5);background-image:-o-linear-gradient(top,#fff,#f5f5f5);background-image:linear-gradient(top,#fff,#f5f5f5);background-repeat:repeat-x;border:1px solid #ddd;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffffff',endColorstr='#f5f5f5',GradientType=0);-webkit-box-shadow:inset 0 1px 0 #fff;-moz-box-shadow:inset 0 1px 0 #fff;box-shadow:inset 0 1px 0 #fff}.breadcrumb li{display:inline-block;*display:inline;text-shadow:0 1px 0 #fff;*zoom:1}.breadcrumb .divider{padding:0 5px;color:#999}.breadcrumb .active a{color:#333}.pagination{height:36px;margin:18px 0}.pagination ul{display:inline-block;*display:inline;margin-bottom:0;margin-left:0;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;*zoom:1;-webkit-box-shadow:0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:0 1px 2px rgba(0,0,0,0.05);box-shadow:0 1px 2px rgba(0,0,0,0.05)}.pagination li{display:inline}.pagination a{float:left;padding:0 14px;line-height:34px;text-decoration:none;border:1px solid #ddd;border-left-width:0}.pagination a:hover,.pagination .active a{background-color:#f5f5f5}.pagination .active a{color:#999;cursor:default}.pagination .disabled span,.pagination .disabled a,.pagination .disabled a:hover{color:#999;cursor:default;background-color:transparent}.pagination li:first-child a{border-left-width:1px;-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.pagination li:last-child a{-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.pagination-centered{text-align:center}.pagination-right{text-align:right}.pager{margin-bottom:18px;margin-left:0;text-align:center;list-style:none;*zoom:1}.pager:before,.pager:after{display:table;content:""}.pager:after{clear:both}.pager li{display:inline}.pager a{display:inline-block;padding:5px 14px;background-color:#fff;border:1px solid #ddd;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px}.pager a:hover{text-decoration:none;background-color:#f5f5f5}.pager .next a{float:right}.pager .previous a{float:left}.pager .disabled a,.pager .disabled a:hover{color:#999;cursor:default;background-color:#fff}.modal-open .dropdown-menu{z-index:2050}.modal-open .dropdown.open{*z-index:2050}.modal-open .popover{z-index:2060}.modal-open .tooltip{z-index:2070}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop,.modal-backdrop.fade.in{opacity:.8;filter:alpha(opacity=80)}.modal{position:fixed;top:50%;left:50%;z-index:1050;width:560px;margin:-250px 0 0 -280px;overflow:auto;background-color:#fff;border:1px solid #999;border:1px solid rgba(0,0,0,0.3);*border:1px solid #999;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 3px 7px rgba(0,0,0,0.3);-moz-box-shadow:0 3px 7px rgba(0,0,0,0.3);box-shadow:0 3px 7px rgba(0,0,0,0.3);-webkit-background-clip:padding-box;-moz-background-clip:padding-box;background-clip:padding-box}.modal.fade{top:-25%;-webkit-transition:opacity .3s linear,top .3s ease-out;-moz-transition:opacity .3s linear,top .3s ease-out;-ms-transition:opacity .3s linear,top .3s ease-out;-o-transition:opacity .3s linear,top .3s ease-out;transition:opacity .3s linear,top .3s ease-out}.modal.fade.in{top:50%}.modal-header{padding:9px 15px;border-bottom:1px solid #eee}.modal-header .close{margin-top:2px}.modal-body{max-height:400px;padding:15px;overflow-y:auto}.modal-form{margin-bottom:0}.modal-footer{padding:14px 15px 15px;margin-bottom:0;text-align:right;background-color:#f5f5f5;border-top:1px solid #ddd;-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px;*zoom:1;-webkit-box-shadow:inset 0 1px 0 #fff;-moz-box-shadow:inset 0 1px 0 #fff;box-shadow:inset 0 1px 0 #fff}.modal-footer:before,.modal-footer:after{display:table;content:""}.modal-footer:after{clear:both}.modal-footer .btn+.btn{margin-bottom:0;margin-left:5px}.modal-footer .btn-group .btn+.btn{margin-left:-1px}.tooltip{position:absolute;z-index:1020;display:block;padding:5px;font-size:11px;opacity:0;filter:alpha(opacity=0);visibility:visible}.tooltip.in{opacity:.8;filter:alpha(opacity=80)}.tooltip.top{margin-top:-2px}.tooltip.right{margin-left:2px}.tooltip.bottom{margin-top:2px}.tooltip.left{margin-left:-2px}.tooltip.top .tooltip-arrow{bottom:0;left:50%;margin-left:-5px;border-top:5px solid #000;border-right:5px solid transparent;border-left:5px solid transparent}.tooltip.left .tooltip-arrow{top:50%;right:0;margin-top:-5px;border-top:5px solid transparent;border-bottom:5px solid transparent;border-left:5px solid #000}.tooltip.bottom .tooltip-arrow{top:0;left:50%;margin-left:-5px;border-right:5px solid transparent;border-bottom:5px solid #000;border-left:5px solid transparent}.tooltip.right .tooltip-arrow{top:50%;left:0;margin-top:-5px;border-top:5px solid transparent;border-right:5px solid #000;border-bottom:5px solid transparent}.tooltip-inner{max-width:200px;padding:3px 8px;color:#fff;text-align:center;text-decoration:none;background-color:#000;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.tooltip-arrow{position:absolute;width:0;height:0}.popover{position:absolute;top:0;left:0;z-index:1010;display:none;padding:5px}.popover.top{margin-top:-5px}.popover.right{margin-left:5px}.popover.bottom{margin-top:5px}.popover.left{margin-left:-5px}.popover.top .arrow{bottom:0;left:50%;margin-left:-5px;border-top:5px solid #000;border-right:5px solid transparent;border-left:5px solid transparent}.popover.right .arrow{top:50%;left:0;margin-top:-5px;border-top:5px solid transparent;border-right:5px solid #000;border-bottom:5px solid transparent}.popover.bottom .arrow{top:0;left:50%;margin-left:-5px;border-right:5px solid transparent;border-bottom:5px solid #000;border-left:5px solid transparent}.popover.left .arrow{top:50%;right:0;margin-top:-5px;border-top:5px solid transparent;border-bottom:5px solid transparent;border-left:5px solid #000}.popover .arrow{position:absolute;width:0;height:0}.popover-inner{width:280px;padding:3px;overflow:hidden;background:#000;background:rgba(0,0,0,0.8);-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 3px 7px rgba(0,0,0,0.3);-moz-box-shadow:0 3px 7px rgba(0,0,0,0.3);box-shadow:0 3px 7px rgba(0,0,0,0.3)}.popover-title{padding:9px 15px;line-height:1;background-color:#f5f5f5;border-bottom:1px solid #eee;-webkit-border-radius:3px 3px 0 0;-moz-border-radius:3px 3px 0 0;border-radius:3px 3px 0 0}.popover-content{padding:14px;background-color:#fff;-webkit-border-radius:0 0 3px 3px;-moz-border-radius:0 0 3px 3px;border-radius:0 0 3px 3px;-webkit-background-clip:padding-box;-moz-background-clip:padding-box;background-clip:padding-box}.popover-content p,.popover-content ul,.popover-content ol{margin-bottom:0}.thumbnails{margin-left:-20px;list-style:none;*zoom:1}.thumbnails:before,.thumbnails:after{display:table;content:""}.thumbnails:after{clear:both}.row-fluid .thumbnails{margin-left:0}.thumbnails>li{float:left;margin-bottom:18px;margin-left:20px}.thumbnail{display:block;padding:4px;line-height:1;border:1px solid #ddd;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:0 1px 1px rgba(0,0,0,0.075);box-shadow:0 1px 1px rgba(0,0,0,0.075)}a.thumbnail:hover{border-color:#08c;-webkit-box-shadow:0 1px 4px rgba(0,105,214,0.25);-moz-box-shadow:0 1px 4px rgba(0,105,214,0.25);box-shadow:0 1px 4px rgba(0,105,214,0.25)}.thumbnail>img{display:block;max-width:100%;margin-right:auto;margin-left:auto}.thumbnail .caption{padding:9px}.label,.badge{font-size:10.998px;font-weight:bold;line-height:14px;color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);white-space:nowrap;vertical-align:baseline;background-color:#999}.label{padding:1px 4px 2px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.badge{padding:1px 9px 2px;-webkit-border-radius:9px;-moz-border-radius:9px;border-radius:9px}a.label:hover,a.badge:hover{color:#fff;text-decoration:none;cursor:pointer}.label-important,.badge-important{background-color:#b94a48}.label-important[href],.badge-important[href]{background-color:#953b39}.label-warning,.badge-warning{background-color:#f89406}.label-warning[href],.badge-warning[href]{background-color:#c67605}.label-success,.badge-success{background-color:#468847}.label-success[href],.badge-success[href]{background-color:#356635}.label-info,.badge-info{background-color:#3a87ad}.label-info[href],.badge-info[href]{background-color:#2d6987}.label-inverse,.badge-inverse{background-color:#333}.label-inverse[href],.badge-inverse[href]{background-color:#1a1a1a}@-webkit-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-moz-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-ms-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-o-keyframes progress-bar-stripes{from{background-position:0 0}to{background-position:40px 0}}@keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}.progress{height:18px;margin-bottom:18px;overflow:hidden;background-color:#f7f7f7;background-image:-moz-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-ms-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-webkit-gradient(linear,0 0,0 100%,from(#f5f5f5),to(#f9f9f9));background-image:-webkit-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-o-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:linear-gradient(top,#f5f5f5,#f9f9f9);background-repeat:repeat-x;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#f5f5f5',endColorstr='#f9f9f9',GradientType=0);-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1)}.progress .bar{width:0;height:18px;font-size:12px;color:#fff;text-align:center;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#0e90d2;background-image:-moz-linear-gradient(top,#149bdf,#0480be);background-image:-webkit-gradient(linear,0 0,0 100%,from(#149bdf),to(#0480be));background-image:-webkit-linear-gradient(top,#149bdf,#0480be);background-image:-o-linear-gradient(top,#149bdf,#0480be);background-image:linear-gradient(top,#149bdf,#0480be);background-image:-ms-linear-gradient(top,#149bdf,#0480be);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#149bdf',endColorstr='#0480be',GradientType=0);-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-moz-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-webkit-box-sizing:border-box;-moz-box-sizing:border-box;-ms-box-sizing:border-box;box-sizing:border-box;-webkit-transition:width .6s ease;-moz-transition:width .6s ease;-ms-transition:width .6s ease;-o-transition:width .6s ease;transition:width .6s ease}.progress-striped .bar{background-color:#149bdf;background-image:-o-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);-webkit-background-size:40px 40px;-moz-background-size:40px 40px;-o-background-size:40px 40px;background-size:40px 40px}.progress.active .bar{-webkit-animation:progress-bar-stripes 2s linear infinite;-moz-animation:progress-bar-stripes 2s linear infinite;-ms-animation:progress-bar-stripes 2s linear infinite;-o-animation:progress-bar-stripes 2s linear infinite;animation:progress-bar-stripes 2s linear infinite}.progress-danger .bar{background-color:#dd514c;background-image:-moz-linear-gradient(top,#ee5f5b,#c43c35);background-image:-ms-linear-gradient(top,#ee5f5b,#c43c35);background-image:-webkit-gradient(linear,0 0,0 100%,from(#ee5f5b),to(#c43c35));background-image:-webkit-linear-gradient(top,#ee5f5b,#c43c35);background-image:-o-linear-gradient(top,#ee5f5b,#c43c35);background-image:linear-gradient(top,#ee5f5b,#c43c35);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ee5f5b',endColorstr='#c43c35',GradientType=0)}.progress-danger.progress-striped .bar{background-color:#ee5f5b;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-success .bar{background-color:#5eb95e;background-image:-moz-linear-gradient(top,#62c462,#57a957);background-image:-ms-linear-gradient(top,#62c462,#57a957);background-image:-webkit-gradient(linear,0 0,0 100%,from(#62c462),to(#57a957));background-image:-webkit-linear-gradient(top,#62c462,#57a957);background-image:-o-linear-gradient(top,#62c462,#57a957);background-image:linear-gradient(top,#62c462,#57a957);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#62c462',endColorstr='#57a957',GradientType=0)}.progress-success.progress-striped .bar{background-color:#62c462;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-info .bar{background-color:#4bb1cf;background-image:-moz-linear-gradient(top,#5bc0de,#339bb9);background-image:-ms-linear-gradient(top,#5bc0de,#339bb9);background-image:-webkit-gradient(linear,0 0,0 100%,from(#5bc0de),to(#339bb9));background-image:-webkit-linear-gradient(top,#5bc0de,#339bb9);background-image:-o-linear-gradient(top,#5bc0de,#339bb9);background-image:linear-gradient(top,#5bc0de,#339bb9);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#5bc0de',endColorstr='#339bb9',GradientType=0)}.progress-info.progress-striped .bar{background-color:#5bc0de;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-warning .bar{background-color:#faa732;background-image:-moz-linear-gradient(top,#fbb450,#f89406);background-image:-ms-linear-gradient(top,#fbb450,#f89406);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fbb450),to(#f89406));background-image:-webkit-linear-gradient(top,#fbb450,#f89406);background-image:-o-linear-gradient(top,#fbb450,#f89406);background-image:linear-gradient(top,#fbb450,#f89406);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fbb450',endColorstr='#f89406',GradientType=0)}.progress-warning.progress-striped .bar{background-color:#fbb450;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.accordion{margin-bottom:18px}.accordion-group{margin-bottom:2px;border:1px solid #e5e5e5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.accordion-heading{border-bottom:0}.accordion-heading .accordion-toggle{display:block;padding:8px 15px}.accordion-toggle{cursor:pointer}.accordion-inner{padding:9px 15px;border-top:1px solid #e5e5e5}.carousel{position:relative;margin-bottom:18px;line-height:1}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel .item{position:relative;display:none;-webkit-transition:.6s ease-in-out left;-moz-transition:.6s ease-in-out left;-ms-transition:.6s ease-in-out left;-o-transition:.6s ease-in-out left;transition:.6s ease-in-out left}.carousel .item>img{display:block;line-height:1}.carousel .active,.carousel .next,.carousel .prev{display:block}.carousel .active{left:0}.carousel .next,.carousel .prev{position:absolute;top:0;width:100%}.carousel .next{left:100%}.carousel .prev{left:-100%}.carousel .next.left,.carousel .prev.right{left:0}.carousel .active.left{left:-100%}.carousel .active.right{left:100%}.carousel-control{position:absolute;top:40%;left:15px;width:40px;height:40px;margin-top:-20px;font-size:60px;font-weight:100;line-height:30px;color:#fff;text-align:center;background:#222;border:3px solid #fff;-webkit-border-radius:23px;-moz-border-radius:23px;border-radius:23px;opacity:.5;filter:alpha(opacity=50)}.carousel-control.right{right:15px;left:auto}.carousel-control:hover{color:#fff;text-decoration:none;opacity:.9;filter:alpha(opacity=90)}.carousel-caption{position:absolute;right:0;bottom:0;left:0;padding:10px 15px 5px;background:#333;background:rgba(0,0,0,0.75)}.carousel-caption h4,.carousel-caption p{color:#fff}.hero-unit{padding:60px;margin-bottom:30px;background-color:#eee;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.hero-unit h1{margin-bottom:0;font-size:60px;line-height:1;letter-spacing:-1px;color:inherit}.hero-unit p{font-size:18px;font-weight:200;line-height:27px;color:inherit}.pull-right{float:right}.pull-left{float:left}.hide{display:none}.show{display:block}.invisible{visibility:hidden}
diff --git a/euscanwww/htdocs/css/style.css b/euscanwww/htdocs/css/style.css
index 556f02f..4191820 100644
--- a/euscanwww/htdocs/css/style.css
+++ b/euscanwww/htdocs/css/style.css
@@ -1,86 +1,67 @@
-body
-{
-  margin: 0;
-  padding: 0;
-  font-size: 0.8em;
-  font-family: Dejavu, Verdana, "Bitstream Vera Sans", "Lucida Grande", "Trebuchet MS", sans-serif;
-  color: #535353;
-  background: #D2D0D4;
-  }
-
-img
-{
-  border: none;
-  }
-
-h1
-{
-  margin-top: 0;
-  color: #369;
-  font-size: 1.6em;
+body {
+    font-size: 0.8em;
+    font-family: Dejavu, Verdana, "Bitstream Vera Sans", "Lucida Grande", "Trebuchet MS", sans-serif;
+    color: #535353;
+    background: #D2D0D4;
 }
 
-a:link, a:visited
-{
-  text-decoration: none;
-  font-weight: bold;
-  color: #ff8c00;
-  }
+img {
+    border: none;
+}
 
-a:hover, a:active
-{
-  font-weight: bold;
-  color: #ff4500;
-  text-decoration: underline;
-  }
+h1 {
+    margin-top: 0;
+    color: #333;
+    font-size: 1.6em;
+    width: 20em;
+    margin-left: auto;
+    margin-right: auto;
+}
+
+a:link, a:visited {
+    text-decoration: none;
+    font-weight: bold;
+    color: #ff8c00;
+}
+
+a:hover, a:active {
+    font-weight: bold;
+    color: #ff4500;
+    text-decoration: underline;
+}
 
 #header {
-    /* background: url(http://packages.gentoo.org/media/packages_gentoo_logo.jpg) no-repeat;
-    background-color: #46347C; */
     width: 100%;
     height: 30px;
     margin: 0;
     padding: 0;
     background: url(../img/gentoo-header-bar-bg.png) repeat-x;
+    margin-bottom: 20px;
 }
 
-#header #logo
-{
+#header #logo {
     float: left;
 }
 
 #content {
-    background: #F0F0F0;
-    padding: 10px;
-    margin-top: 10px;
-/*    margin-right: auto;
-    margin-left: auto;
-*/
-    margin-left: 20px;
-    /* margin-left: 20%; */
-    margin-right: 250px;
-    /* max-width: 60em; */
-
     border: 1px solid #67539B;
     background: #FEFEFE;
 
-    -webkit-border-radius: 10px;
-    -moz-border-radius: 10px;
-    border-radius: 10px;
+    padding: 20px;
+
+    min-height: 350px;
 }
 
 .menu {
-    -webkit-border-radius: 10px;
-    -moz-border-radius: 10px;
-    border-radius: 10px;
     border: 1px solid #67539B;
-}
-
-.menu {
-    width:100%;
-    margin-top: 30px;
     background: #8076A1;
     color: #FFF;
+    padding-top: 5px;
+    padding-bottom: 5px;
+}
+
+.notfirst-menu {
+    margin-top: 10px;
 }
 
 .menu a {
@@ -103,29 +84,7 @@ a:hover, a:active
     border: none;
 }
 
-#menus {
-    width: 200px;
-    position:absolute;
-    top:30px;
-    right:20px;
-
-/*
-    position: fixed;
-    right: 10px;
-    top: 2em;
-    width: 20%;
-*/
-}
-
-/*
-code, pre{
-    background-color:transparent;
-    font-family:"Courier New",Courier,monospace;
-    font-size:small;
-}
-*/
-
-a{
+a {
     color: #3F4C66;
 }
 
@@ -142,7 +101,7 @@ abbr:hover {
     text-decoration: underline;
 }
 
-pre{
+pre {
     border-left:5px solid;
     padding:0.5em 1em;
     margin-left:2em;
@@ -158,18 +117,6 @@ dd {
     padding: 5px;
 }
 
-
-h1 {
-    color: #000;
-    width: 20em;
-    margin-left: auto;
-    margin-right: auto;
-}
-
-.ok {
-    color:#15B100;
-}
-
 hr {
     margin: 0.3em 1em 0.3em 1em;
     height: 1px;
@@ -204,7 +151,7 @@ th {
     font-size: 0.8em;
     width: 100%;
     margin: 0;
-    margin-top: 10px;
+    margin-top: 20px;
     color: #FFF;
 }
 
@@ -213,8 +160,7 @@ th {
     margin: 5px;
 }
 
-.err,.ok,.inf
-{
+.err, .ok, .inf {
     margin: 5px;
     margin-left: auto;
     margin-right: auto;
@@ -281,3 +227,9 @@ th {
 .hide {
     display: none;
 }
+
+.rounded {
+    -webkit-border-radius: 10px;
+    -moz-border-radius: 10px;
+    border-radius: 10px;
+}

From fc7f9290389925bab77aeb20d214517e0e2ad0a1 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Fri, 29 Jun 2012 17:25:45 +0200
Subject: [PATCH 37/37] euscanwww: Fixed broken tasks

Signed-off-by: volpino <fox91@anche.no>
---
 .../djeuscan/processing/scan_metadata.py      |  2 +-
 euscanwww/djeuscan/processing/scan_portage.py | 54 ++++++++++---------
 .../djeuscan/processing/scan_upstream.py      | 37 +++++++++----
 euscanwww/djeuscan/tasks.py                   | 33 +++++++-----
 euscanwww/euscanwww/settings.py               |  2 +-
 pym/euscan/out.py                             | 13 ++++-
 pym/euscan/scan.py                            | 10 ++--
 7 files changed, 98 insertions(+), 53 deletions(-)

diff --git a/euscanwww/djeuscan/processing/scan_metadata.py b/euscanwww/djeuscan/processing/scan_metadata.py
index 02a8e84..c834327 100644
--- a/euscanwww/djeuscan/processing/scan_metadata.py
+++ b/euscanwww/djeuscan/processing/scan_metadata.py
@@ -137,7 +137,7 @@ class ScanMetadata(object):
 
 def scan_metadata(packages=None, logger=None):
     scan_handler = ScanMetadata(logger=logger)
-    if packages is None:
+    if not packages:
         packages = Package.objects.all()
 
     for pkg in packages:
diff --git a/euscanwww/djeuscan/processing/scan_portage.py b/euscanwww/djeuscan/processing/scan_portage.py
index 14d7a8e..29f5974 100644
--- a/euscanwww/djeuscan/processing/scan_portage.py
+++ b/euscanwww/djeuscan/processing/scan_portage.py
@@ -15,11 +15,10 @@ from djeuscan.models import Package, Version, VersionLog
 
 class ScanPortage(object):
     def __init__(self, logger=None, no_log=False, purge_packages=False,
-                 purge_versions=False, kill_versions=False):
+                 kill_versions=False):
         self.logger = logger or FakeLogger()
         self.no_log = no_log
         self.purge_packages = purge_packages
-        self.purge_versions = purge_versions
         self.kill_versions = kill_versions
 
         self.style = color_style()
@@ -84,8 +83,13 @@ class ScanPortage(object):
 
         return self._overlays
 
-    @commit_on_success
     def scan(self, query=None):
+        if self.purge_packages:
+            with commit_on_success():
+                for package in Package.objects.all():
+                    self.logger.info('- [p] %s' % (package))
+                    package.delete()
+
         cmd = ['eix', '--xml', '--pure-packages', '-x']
         if query:
             cmd.extend(['--exact', query])
@@ -97,7 +101,6 @@ class ScanPortage(object):
 
         output = subprocess.Popen(cmd, stdout=subprocess.PIPE).\
             communicate()[0]
-        output = output.strip().strip('\n')
 
         if len(output) == 0:
             if not query:
@@ -124,24 +127,26 @@ class ScanPortage(object):
                 cat = category_tag.getAttribute("name")
                 pkg = package_tag.getAttribute("name")
                 homepage_tags = package_tag.getElementsByTagName("homepage")
-                homepage = homepage_tags[0].firstChild.nodeValue \
-                           if homepage_tags else ""
+                try:
+                    homepage = homepage_tags[0].firstChild.nodeValue
+                except (IndexError, AttributeError):
+                    homepage = ""
                 desc_tags = package_tag.getElementsByTagName("description")
-                desc = desc_tags[0].firstChild.nodeValue if desc_tags else ""
+                try:
+                    desc = desc_tags[0].firstChild.nodeValue
+                except (IndexError, AttributeError):
+                    desc = ""
 
-                package = self.store_package(cat, pkg, homepage, desc)
+                with commit_on_success():
+                    package = self.store_package(cat, pkg, homepage, desc)
 
-                for version_tag in package_tag.getElementsByTagName("version"):
-                    cpv = "%s/%s-%s" % (cat, pkg,
-                                        version_tag.getAttribute("id"))
-                    slot = version_tag.getAttribute("slot")
-                    overlay = version_tag.getAttribute("overlay")
-                    self.store_version(package, cpv, slot, overlay)
-
-        if self.purge_packages and not query:
-            for package in Package.objects.all():
-                self.logger.info('- [p] %s' % (package))
-                package.delete()
+                    for version_tag in package_tag.\
+                                       getElementsByTagName("version"):
+                        cpv = "%s/%s-%s" % (cat, pkg,
+                                            version_tag.getAttribute("id"))
+                        slot = version_tag.getAttribute("slot")
+                        overlay = version_tag.getAttribute("overlay")
+                        self.store_version(package, cpv, slot, overlay)
 
     def store_package(self, cat, pkg, homepage, description):
         created = False
@@ -232,7 +237,7 @@ class ScanPortage(object):
 
 
 @commit_on_success
-def purge_versions(logger=None, nolog=False):
+def do_purge_versions(logger=None, no_log=False):
     logger = logger or FakeLogger()
 
     # For each dead versions
@@ -246,7 +251,7 @@ def purge_versions(logger=None, nolog=False):
 
         logger.info('- [v] %s' % (version))
 
-        if nolog:
+        if no_log:
             continue
 
         VersionLog.objects.create(
@@ -274,7 +279,6 @@ def scan_portage(packages=None, no_log=False, purge_packages=False,
         logger=logger,
         no_log=no_log,
         purge_packages=purge_packages,
-        purge_versions=purge_versions,
         kill_versions=kill_versions,
     )
 
@@ -288,16 +292,16 @@ def scan_portage(packages=None, no_log=False, purge_packages=False,
             scan_handler.cache_store_version(version)
         logger.info('done')
 
-    if packages is None:
+    if not packages:
         scan_handler.scan()
     else:
         for pkg in packages:
             if isinstance(pkg, Package):
-                scan_handler.scan('%s/%s' % (pkg.category, pkg.name), pkg)
+                scan_handler.scan('%s/%s' % (pkg.category, pkg.name))
             else:
                 scan_handler.scan(pkg)
 
     if purge_versions:
-        purge_versions(logger=logger, no_log=no_log)
+        do_purge_versions(logger=logger, no_log=no_log)
     logger.info('Done.')
     return True
diff --git a/euscanwww/djeuscan/processing/scan_upstream.py b/euscanwww/djeuscan/processing/scan_upstream.py
index 827ad91..981af59 100644
--- a/euscanwww/djeuscan/processing/scan_upstream.py
+++ b/euscanwww/djeuscan/processing/scan_upstream.py
@@ -16,6 +16,7 @@ class ScanUpstream(object):
 
     def scan(self, package):
         CONFIG["format"] = "dict"
+        output.clean()
         output.set_query(package)
 
         euscan_scan_upstream(package)
@@ -34,7 +35,14 @@ class ScanUpstream(object):
             obj = self.store_package(cpv)
 
             for res in out[package]["result"]:
-                self.store_version(obj, res["version"], " ".join(res["urls"]))
+                self.store_version(
+                    obj,
+                    res["version"],
+                    " ".join(res["urls"]),
+                    res["type"],
+                    res["handler"],
+                    res["confidence"],
+                )
 
             self.store_result(obj, out_json, scan_time, ebuild)
 
@@ -66,10 +74,17 @@ class ScanUpstream(object):
 
         return obj
 
-    def store_version(self, package, ver, url):
+    def store_version(self, package, ver, url, version_type, handler,
+                      confidence):
         obj, created = Version.objects.get_or_create(
-            package=package, slot='', revision='r0', version=ver, overlay='',
-            defaults={"alive": True, "urls": url, "packaged": False}
+            package=package,
+            slot='',
+            revision='r0',
+            version=ver,
+            overlay='',
+            defaults={"alive": True, "urls": url, "packaged": False,
+                      "version_type": version_type, "handler": handler,
+                      "confidence": confidence}
         )
         if not created:
             obj.alive = True
@@ -97,7 +112,7 @@ class ScanUpstream(object):
 
 
 @commit_on_success
-def purge_versions(logger=None):
+def do_purge_versions(logger=None):
     logger = logger or FakeLogger()
 
     # For each dead versions
@@ -126,17 +141,21 @@ def scan_upstream(packages=None, purge_versions=False,
 
     logger.info('Scanning upstream...')
 
-    if packages is None:
+    if not packages:
         packages = Package.objects.all()
 
+    result = True
+
     for pkg in packages:
         if isinstance(pkg, Package):
-            result = scan_handler.scan('%s/%s' % (pkg.category, pkg.name))
+            curr = scan_handler.scan('%s/%s' % (pkg.category, pkg.name))
         else:
-            result = scan_handler.scan(pkg)
+            curr = scan_handler.scan(pkg)
+        if not curr:
+            result = False
 
     if purge_versions:
-        purge_versions(logger=logger)
+        do_purge_versions(logger=logger)
 
     logger.info('Done.')
     return result
diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index 4f6842e..3fcbcae 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -58,7 +58,7 @@ def _run_in_chunks(task, packages, kwargs=None,
         job = TaskSet(tasks=tasks)
         result = job.apply_async()
         # TODO: understand why this causes timeout
-        output.extend(list(result.join(timeout=3600)))
+        output.extend(list(result.join()))
     return output
 
 
@@ -102,7 +102,7 @@ def scan_metadata_list_task(query):
     Runs a parallel metadata scan for packages in the query list (space
     separated string). Task used only from the web interface.
     """
-    _run_in_chunks(_scan_metadata_task, [p for p in query.split()])
+    return _run_in_chunks(_scan_metadata_task, [p for p in query.split()])
 
 
 @task
@@ -110,7 +110,7 @@ def scan_metadata_all_task():
     """
     Runs a parallel metadata scan for all packages
     """
-    _run_in_chunks(_scan_metadata_task, Package.objects.all())
+    return _run_in_chunks(_scan_metadata_task, Package.objects.all())
 
 
 @task
@@ -120,8 +120,11 @@ def _scan_portage_task(packages, no_log=False, purge_packages=False,
     Scans portage for the given set of packages
     """
     logger = _scan_portage_task.get_logger()
-    logger.info("Starting portage scanning subtask for %d packages...",
-                len(packages))
+    if packages:
+        logger.info("Starting portage scanning subtask for %d packages...",
+                    len(packages))
+    else:
+        logger.info("Starting portage scanning for all packages...")
 
     result = scan_portage(
         packages=packages,
@@ -145,18 +148,23 @@ def scan_portage_list_task(query, no_log=False, purge_packages=False,
     """
     kwargs = {"no_log": no_log, "purge_packages": purge_packages,
               "purge_versions": purge_versions, "prefetch": prefetch}
-    _run_in_chunks(_scan_portage_task, [p for p in query.split()], kwargs)
+    return _run_in_chunks(_scan_portage_task, [p for p in query.split()],
+                          kwargs)
 
 
 @task
 def scan_portage_all_task(no_log=False, purge_packages=False,
                           purge_versions=False, prefetch=False):
     """
-    Runs a parallel portage scan for all packages
+    Runs a syncronous portage scan for all packages
     """
-    kwargs = {"no_log": no_log, "purge_packages": purge_packages,
-              "purge_versions": purge_versions, "prefetch": prefetch}
-    _run_in_chunks(_scan_metadata_task, Package.objects.all(), kwargs)
+    return _scan_portage_task(
+        packages=None,
+        no_log=no_log,
+        purge_packages=purge_packages,
+        purge_versions=purge_versions,
+        prefetch=prefetch,
+    )
 
 
 @task
@@ -187,7 +195,8 @@ def scan_upstream_list_task(query, purge_versions=False):
     """
 
     kwargs = {"purge_versions": purge_versions}
-    _run_in_chunks(_scan_upstream_task, [p for p in query.split()], kwargs)
+    return _run_in_chunks(_scan_upstream_task, [p for p in query.split()],
+                          kwargs)
 
 
 @task
@@ -196,7 +205,7 @@ def scan_upstream_all_task(purge_versions=False):
     Runs a parallel portage scan for all packages
     """
     kwargs = {"purge_versions": purge_versions}
-    _run_in_chunks(_scan_upstream_task, Package.objects.all(), kwargs)
+    return _run_in_chunks(_scan_upstream_task, Package.objects.all(), kwargs)
 
 
 @task
diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py
index 0fe0f89..9a96165 100644
--- a/euscanwww/euscanwww/settings.py
+++ b/euscanwww/euscanwww/settings.py
@@ -235,7 +235,7 @@ CELERY_RESULT_BACKEND = "amqp"
 BROKER_CONNECTION_TIMEOUT = 3600
 CELERYD_CONCURRENCY = 4
 
-TASKS_CONCURRENTLY = 8
+TASKS_CONCURRENTLY = 4
 TASKS_SUBTASK_PACKAGES = 32
 
 # LDAP authentication
diff --git a/pym/euscan/out.py b/pym/euscan/out.py
index 1928980..8a260d3 100644
--- a/pym/euscan/out.py
+++ b/pym/euscan/out.py
@@ -67,12 +67,20 @@ class EOutputMem(EOutput):
         self.out = StringIO()
 
     def getvalue(self):
-        return re.sub("\033\[[0-9;]+m", "", self.out.getvalue())
+        return clean_colors(self.out.getvalue())
 
     def _write(self, f, msg):
         super(EOutputMem, self)._write(self.out, msg)
 
 
+def clean_colors(string):
+    if type(string) is str:
+        string = re.sub("\033\[[0-9;]+m", "", string)
+        string = re.sub(r"\\u001b\[[0-9;]+m", "", string)
+        string = re.sub(r"\x1b\[[0-9;]+m", "", string)
+    return string
+
+
 class EuscanOutput(object):
     """
     Class that handles output for euscan
@@ -141,7 +149,8 @@ class EuscanOutput(object):
 
     def metadata(self, key, value, show=True):
         if self.config["format"]:
-            self.queries[self.current_query]["metadata"][key] = value
+            self.queries[self.current_query]["metadata"][key] = \
+                clean_colors(value)
         elif show:
             print "%s: %s" % (key.capitalize(), value)
 
diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py
index be49b96..fa573f3 100644
--- a/pym/euscan/scan.py
+++ b/pym/euscan/scan.py
@@ -45,9 +45,13 @@ def filter_versions(cp, versions):
 def scan_upstream_urls(cpv, urls, on_progress):
     versions = []
 
-    progress_available = 70
-    num_urls = sum([len(urls[fn]) for fn in urls])
-    progress_increment = progress_available / num_urls
+    if on_progress:
+        progress_available = 70
+        num_urls = sum([len(urls[fn]) for fn in urls])
+        if num_urls > 0:
+            progress_increment = progress_available / num_urls
+        else:
+            progress_increment = 0
 
     for filename in urls:
         for url in urls[filename]: