""" Celery tasks for djeuscan """ from itertools import islice from celery.task import task, group, chord from django.conf import settings from djeuscan.models import Package, RefreshPackageQuery from djeuscan.processing import scan, misc from djeuscan.utils import queryset_iterator class TaskFailedException(Exception): """ Exception for failed tasks """ pass def group_one(task, seq, *args, **kwargs): """ Create a group of tasks, each task handle one element of seq """ tasks = [] for i in seq: tasks.append(task.subtask(args=[seq[i]] + list(args), kwargs=kwargs)) return group(tasks) def group_chunks(task, seq, n, *args, **kwargs): """ Creates a group of tasks, each subtask has elements to handle """ tasks = [] for i in xrange(0, len(seq), n): tasks.append(task.subtask(args=[seq[i:i+n]] + list(args), kwargs=kwargs)) return group(tasks) @task def regen_rrds(): """ Regenerate RRDs """ misc.regen_rrds() return True @task def update_counters(fast=False): """ Updates counters """ logger = update_counters.get_logger() logger.info("Updating counters (fast=%s)...", fast) misc.update_counters(fast=fast) logger.info("Done") return True @task def scan_metadata(packages=[], category=None): """ Scans metadata for the given set of packages """ logger = scan_metadata.get_logger() if packages: logger.info("Starting metadata scan for %d packages...", len(packages)) elif category: logger.info("Starting metadata scan for %s...", category) else: logger.info("Starting metadata scan...") scan.scan_metadata( packages=packages, category=category, logger=logger, ) return True @task def scan_portage(packages=[], category=None, no_log=False, purge_packages=False, purge_versions=False, prefetch=False): """ Scans portage for the given set of packages """ logger = scan_portage.get_logger() if packages: logger.info("Starting portage scan for %d packages...", len(packages)) elif category: logger.info("Starting portage scan for %s...", category) else: logger.info("Starting portage scan...") scan.scan_portage( packages=packages, category=category, no_log=no_log, purge_packages=purge_packages, purge_versions=purge_versions, prefetch=prefetch, logger=logger, ) return True @task def scan_upstream(packages=[], purge_versions=False): """ Scans upstream for the given set of packages """ logger = scan_upstream.get_logger() if len(packages): logger.info("Starting upstream scan subtask for %d packages...", len(packages)) else: logger.info("Starting upstream scan...", len(packages)) scan.scan_upstream( packages=packages, purge_versions=purge_versions, logger=logger, ) return True @task def update_portage_trees(): """ Update portage tree """ logger = update_portage_trees.get_logger() misc.update_portage_trees(logger=logger) return True @task def update_portage(packages=None): ( update_portage_trees.s() | scan_portage.si(purge_packages=True, purge_versions=True, prefetch=True) | #scan_metadata.si() | group_one(scan_metadata, portage.settings.categories) | update_counters.si(fast=False) )() return True @task def update_upstream(): if settings.TASKS_UPSTREAM_GROUPS >= 1: packages = Package.objects.all() scan_upstream_sub = group_chunks(scan_upstream, packages, settings.TASKS_UPSTREAM_GROUPS, purge_versions=True) else: scan_upstream_sub = scan_upstream.si(purge_versions=True) ( scan_upstream_sub | update_counters.si(fast=False) )() return True @task def scan_package(package): scan_portage([package], purge_packages=True, purge_versions=True) scan_metadata([package]) scan_upstream([package]) return True @task(rate_limit="1/m") def scan_package_user(package): scan_package(package) return True @task def consume_refresh_package_request(): """ Satisfies user requests for package refreshing, runs every minute """ try: query = RefreshPackageQuery.objects.all().order_by('-priority')[0] except IndexError: return pkg = query.package query.delete() scan_package_user.delay(pkg) admin_tasks = [ regen_rrds, update_counters, scan_metadata, scan_portage, scan_upstream, update_portage_trees, update_portage, update_upstream, scan_package, ] """ Chunk helpers (chunks can't use keyword arguments) """ @task def scan_metadata_category(category): """ Helper for calling scan_metadata with a category """ scan_metadata(category=category) return True @task def scan_upstream_purge(*packages): """ Helper for calling scan_upstream with purge_versions=True """ scan_upstream(packages, purge_versions=True) return True