diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py index 8dfbeaa..1e66838 100644 --- a/euscanwww/djeuscan/tasks.py +++ b/euscanwww/djeuscan/tasks.py @@ -8,6 +8,8 @@ from celery.task import task, periodic_task from celery.task.schedules import crontab from celery.task.sets import TaskSet +from django.conf import settings + from djeuscan.models import Package, RefreshPackageQuery from djeuscan.processing.regen_rrds import regen_rrds from djeuscan.processing.update_counters import update_counters @@ -32,7 +34,9 @@ def _chunks(it, n): yield [first] + list(islice(it, n - 1)) -def _run_in_chunks(task, packages, kwargs=None, concurrently=8, n=32): +def _run_in_chunks(task, packages, kwargs=None, + concurrently=settings.TASKS_CONCURRENTLY, + n=settings.TASKS_SUBTASK_PACKAGES): """ Launches a TaskSet at a time with subtasks. Each subtask has packages to handle diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py index 116fcd9..0fe0f89 100644 --- a/euscanwww/euscanwww/settings.py +++ b/euscanwww/euscanwww/settings.py @@ -235,6 +235,9 @@ CELERY_RESULT_BACKEND = "amqp" BROKER_CONNECTION_TIMEOUT = 3600 CELERYD_CONCURRENCY = 4 +TASKS_CONCURRENTLY = 8 +TASKS_SUBTASK_PACKAGES = 32 + # LDAP authentication # TODO: Test data - change me! AUTH_LDAP_SERVER_URI = "ldap://localhost"