From 90c7f27b95391ae0151de7fe15568c7c10941f6d Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Mon, 29 Oct 2012 12:48:08 +0100 Subject: [PATCH] djeuscan/processing: fix scan_upstream after scan_portage Signed-off-by: Corentin Chary --- .../management/commands/scan_portage.py | 6 ---- .../processing/misc/update_counters.py | 1 - .../djeuscan/processing/scan/scan_portage.py | 29 +++++++++++-------- .../djeuscan/processing/scan/scan_upstream.py | 2 +- euscanwww/djeuscan/tasks.py | 20 ++++++++----- 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/euscanwww/djeuscan/management/commands/scan_portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py index e70affa..e40778c 100644 --- a/euscanwww/djeuscan/management/commands/scan_portage.py +++ b/euscanwww/djeuscan/management/commands/scan_portage.py @@ -34,11 +34,6 @@ class Command(BaseCommand): dest='purge-versions', default=False, help='Purge old versions'), - make_option('--upstream', - action='store_true', - dest='upstream', - default=False, - help='Scan upstream for updated packages in Portage'), make_option('--no-log', action='store_true', dest='no-log', @@ -71,6 +66,5 @@ class Command(BaseCommand): purge_packages=options["purge-packages"], purge_versions=options["purge-versions"], prefetch=options["prefetch"], - upstream=options["upstream"], logger=logger, ) diff --git a/euscanwww/djeuscan/processing/misc/update_counters.py b/euscanwww/djeuscan/processing/misc/update_counters.py index fdbea9c..392b7b5 100644 --- a/euscanwww/djeuscan/processing/misc/update_counters.py +++ b/euscanwww/djeuscan/processing/misc/update_counters.py @@ -114,7 +114,6 @@ def update_counters(fast=False, nolog=False, logger=None): package.last_version_upstream_id = last_versions_upstream.get( package.id, default )['id'] - package.save() n_packages_gentoo = int(package.n_packaged == package.n_versions) diff --git a/euscanwww/djeuscan/processing/scan/scan_portage.py b/euscanwww/djeuscan/processing/scan/scan_portage.py index 1ce7197..2039d5e 100644 --- a/euscanwww/djeuscan/processing/scan/scan_portage.py +++ b/euscanwww/djeuscan/processing/scan/scan_portage.py @@ -13,23 +13,25 @@ from django.core.management.color import color_style from euscan.version import get_version_type from djeuscan.processing import FakeLogger -from djeuscan.processing.scan.scan_upstream import scan_upstream from djeuscan.models import Package, Version, VersionLog, Category, Overlay class ScanPortage(object): def __init__(self, logger=None, no_log=False, purge_packages=False, - purge_versions=False, upstream=False): + purge_versions=False): self.logger = logger or FakeLogger() self.no_log = no_log self.purge_packages = purge_packages self.purge_versions = purge_versions - self.upstream = upstream self.style = color_style() self._cache = {'packages': {}, 'versions': {}} self._overlays = None + self._updated_packages = set() + + def updated_packages(self): + return list(self._updated_packages) def cache_hash_package(self, category, name): return '%s/%s' % (category, name) @@ -200,13 +202,14 @@ class ScanPortage(object): packages_alive.add("%s/%s" % (cat, pkg)) new_version = False for cpv, slot, overlay, overlay_path in data['versions']: - new_version = new_version or self.store_version( + obj, created = self.store_version( package, cpv, slot, overlay, overlay_path ) + new_version = created or new_version # If the package has at least one new version scan upstream for it - if new_version and self.upstream: - scan_upstream([package], self.purge_versions, self.logger) + if new_version: + self._updated_packages.add(package) self.purge_old_packages(current_packages, packages_alive) self.purge_old_versions() @@ -269,7 +272,7 @@ class ScanPortage(object): # nothing to do (note: it can't be an upstream version because # overlay can't be empty here) if not created: - return False + return obj, created # New version created self.logger.info('+ [v] %s' % (obj)) @@ -292,7 +295,7 @@ class ScanPortage(object): vtype=obj.vtype, ) - return True + return obj, created def purge_old_packages(self, packages, alive): if not self.purge_packages: @@ -336,7 +339,7 @@ class ScanPortage(object): @commit_on_success -def scan_portage(packages=None, category=None, no_log=False, upstream=False, +def scan_portage(packages=None, category=None, no_log=False, purge_packages=False, purge_versions=False, prefetch=False, logger=None): @@ -349,8 +352,7 @@ def scan_portage(packages=None, category=None, no_log=False, upstream=False, logger=logger, no_log=no_log, purge_packages=purge_packages, - purge_versions=purge_versions, - upstream=upstream + purge_versions=purge_versions ) logger.info('Scanning portage tree...') @@ -368,8 +370,10 @@ def scan_portage(packages=None, category=None, no_log=False, upstream=False, scan_handler.cache_store_version(version) logger.info('done') - if not packages: + if not packages and category: scan_handler.scan(category=category) + elif not packages: + scan_handler.scan() else: for pkg in packages: if isinstance(pkg, Package): @@ -394,3 +398,4 @@ def scan_portage(packages=None, category=None, no_log=False, upstream=False, logger.info("+ [o] %s", overlay["overlay"]) logger.info('Done.') + return scan_handler.updated_packages() diff --git a/euscanwww/djeuscan/processing/scan/scan_upstream.py b/euscanwww/djeuscan/processing/scan/scan_upstream.py index 6ec2b76..bc9159b 100644 --- a/euscanwww/djeuscan/processing/scan/scan_upstream.py +++ b/euscanwww/djeuscan/processing/scan/scan_upstream.py @@ -34,7 +34,7 @@ class ScanUpstream(object): "Error while scanning upstream for package %s!", package ) - self.logger.info( + self.logger.debug( "Error %s", out_json ) diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py index 9c93989..93adeac 100644 --- a/euscanwww/djeuscan/tasks.py +++ b/euscanwww/djeuscan/tasks.py @@ -113,7 +113,7 @@ def scan_metadata(packages=[], category=None, populate=False): @task -def scan_portage(packages=[], category=None, upstream=True, +def scan_portage(packages=None, category=None, no_log=False, purge_packages=False, purge_versions=False, prefetch=False): """ @@ -130,17 +130,15 @@ def scan_portage(packages=[], category=None, upstream=True, else: logger.info("Starting portage scan...") - scan.scan_portage( + return scan.scan_portage( packages=packages, category=category, - upstream=upstream, no_log=no_log, purge_packages=purge_packages, purge_versions=purge_versions, prefetch=prefetch, logger=logger, ) - return True @task @@ -180,9 +178,17 @@ def update_portage(packages=None): # Workaround for celery bug when chaining groups update_portage_trees() - scan_portage(packages=[], purge_packages=True, purge_versions=True, - prefetch=True) - scan_metadata(packages=[], populate=True) + updated_packages = scan_portage( + packages=None, + purge_packages=True, + purge_versions=True, + prefetch=True + ) + scan_metadata(packages=None, populate=True) + if updated_packages: + group_chunks(scan_upstream, updated_packages, + settings.TASKS_UPSTREAM_GROUPS, + purge_versions=True)() update_counters(fast=False) """ Currently broken