From 4246c544ada320d0f58ef002a0e25b7e9223f1d2 Mon Sep 17 00:00:00 2001
From: volpino <fox91@anche.no>
Date: Sat, 9 Jun 2012 16:12:17 +0200
Subject: [PATCH] euscanwww: bug found, celery timeouts on join. Commented out

Signed-off-by: volpino <fox91@anche.no>
---
 euscanwww/djeuscan/tasks.py     | 18 +++++++++++++++---
 euscanwww/euscanwww/settings.py |  2 ++
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py
index 77064b0..425467b 100644
--- a/euscanwww/djeuscan/tasks.py
+++ b/euscanwww/djeuscan/tasks.py
@@ -18,6 +18,13 @@ from djeuscan.management.commands.scan_upstream import ScanUpstream, \
     purge_versions as scan_upstream_purge
 
 
+class TaskFailedException(Exception):
+    """
+    Exception for failed tasks
+    """
+    pass
+
+
 def _launch_command(cmd):
     fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
@@ -38,7 +45,8 @@ def _run_in_chunks(task, iterable, n=32):
             for args in chunk
         ])
         result = job.apply_async()
-        output.extend(list(result.join()))
+        # TODO: understand why this causes timeout
+        #output.extend(list(result.join(timeout=3600)))
     return output
 
 
@@ -114,7 +122,10 @@ def scan_upstream_task(query):
     logger.info("Starting upstream scanning for package %s ...", query)
 
     scan_upstream = ScanUpstream()
-    return scan_upstream.scan(query)
+    result = scan_upstream.scan(query)
+    if not result:
+        raise TaskFailedException("Couldn't scan upstream for this package")
+    return result
 
 
 @task
@@ -127,7 +138,8 @@ def scan_upstream_all_task(purge=False):
     output = _run_in_chunks(
         scan_upstream_task,
         [('%s/%s' % (pkg.category, pkg.name), )
-         for pkg in Package.objects.all()]
+         for pkg in Package.objects.all()],
+        n=16
     )
 
     if purge:
diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py
index 46182c0..d5f0ff6 100644
--- a/euscanwww/euscanwww/settings.py
+++ b/euscanwww/euscanwww/settings.py
@@ -213,6 +213,8 @@ import djcelery
 djcelery.setup_loader()
 BROKER_URL = "amqp://guest:guest@localhost:5672//"
 CELERY_RESULT_BACKEND = "amqp"
+BROKER_CONNECTION_TIMEOUT = 600
+CELERYD_CONCURRENCY = 4
 
 
 try: