From ba805536ad66aa196fbce88f20a1146fb2c0d685 Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Mon, 2 Jul 2012 11:19:49 +0200 Subject: [PATCH] euscanwww: add --category to scan_metadata, remove homepage and description Signed-off-by: Corentin Chary --- .../management/commands/scan_metadata.py | 13 ++- .../djeuscan/processing/scan_metadata.py | 108 +++++++++--------- euscanwww/djeuscan/processing/scan_portage.py | 2 +- 3 files changed, 63 insertions(+), 60 deletions(-) diff --git a/euscanwww/djeuscan/management/commands/scan_metadata.py b/euscanwww/djeuscan/management/commands/scan_metadata.py index 5decb70..3d98a36 100644 --- a/euscanwww/djeuscan/management/commands/scan_metadata.py +++ b/euscanwww/djeuscan/management/commands/scan_metadata.py @@ -19,6 +19,11 @@ class Command(BaseCommand): dest='all', default=False, help='Scan all packages'), + make_option('--category', + action='store', + dest='category', + default=None, + help='Scan only this category'), ) args = '' help = 'Scans metadata and fills database' @@ -26,7 +31,7 @@ class Command(BaseCommand): def handle(self, *args, **options): set_verbosity_level(logger, options.get("verbosity", 1)) - if options['all']: + if options['all'] or options['category']: packages = None elif len(args): @@ -34,4 +39,8 @@ class Command(BaseCommand): else: packages = [pkg[:-1] for pkg in sys.stdin.readlines()] - scan_metadata(packages=packages, logger=logger) + scan_metadata( + packages=packages, + category=options['category'], + logger=logger + ) diff --git a/euscanwww/djeuscan/processing/scan_metadata.py b/euscanwww/djeuscan/processing/scan_metadata.py index c834327..263f4c0 100644 --- a/euscanwww/djeuscan/processing/scan_metadata.py +++ b/euscanwww/djeuscan/processing/scan_metadata.py @@ -1,3 +1,5 @@ +import os.path + from gentoolkit.query import Query from gentoolkit.errors import GentoolkitFatalError @@ -8,17 +10,19 @@ from django.core.exceptions import ValidationError from djeuscan.models import Package, Herd, Maintainer from djeuscan.processing import FakeLogger - class ScanMetadata(object): def __init__(self, logger=None): self.style = color_style() self.logger = logger or FakeLogger() - @commit_on_success def scan(self, query=None, obj=None): - matches = Query(query).find( - include_masked=True, - in_installed=False, + matches = Query(query).smart_find( + in_installed=True, + in_porttree=True, + in_overlay=True, + include_masked=True, + show_progress=False, + no_matches_fatal=False, ) if not matches: @@ -39,65 +43,52 @@ class ScanMetadata(object): else: created = False - try: - obj.homepage = pkg.environment("HOMEPAGE") - obj.description = pkg.environment("DESCRIPTION") - except GentoolkitFatalError, err: - self.logger.error( - self.style.ERROR( - "Gentoolkit fatal error: '%s'" % str(err) - ) - ) - if created: self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name)) - if pkg.metadata: - herds = dict( - [(herd[0], herd) for herd in pkg.metadata.herds(True)] - ) - maintainers = dict( - [(m.email, m) for m in pkg.metadata.maintainers()] - ) + if not pkg.metadata: + return - existing_herds = [h.herd for h in obj.herds.all()] - new_herds = set(herds.keys()).difference(existing_herds) - old_herds = set(existing_herds).difference(herds.keys()) + herds = dict( + [(herd[0], herd) for herd in pkg.metadata.herds(True)] + ) + maintainers = dict( + [(m.email, m) for m in pkg.metadata.maintainers()] + ) - existing_maintainers = [m.email for m in obj.maintainers.all()] - new_maintainers = set( - maintainers.keys()).difference(existing_maintainers - ) - old_maintainers = set( - existing_maintainers).difference(maintainers.keys() - ) + existing_herds = [h.herd for h in obj.herds.all()] + new_herds = set(herds.keys()).difference(existing_herds) + old_herds = set(existing_herds).difference(herds.keys()) - for herd in obj.herds.all(): - if herd.herd in old_herds: - obj.herds.remove(herd) + existing_maintainers = [m.email for m in obj.maintainers.all()] + new_maintainers = set(maintainers.keys()).difference(existing_maintainers) + old_maintainers = set(existing_maintainers).difference(maintainers.keys()) - for herd in new_herds: - herd = self.store_herd(*herds[herd]) - obj.herds.add(herd) + for herd in obj.herds.all(): + if herd.herd in old_herds: + obj.herds.remove(herd) - for maintainer in obj.maintainers.all(): - if maintainer.email in old_maintainers: - obj.maintainers.remove(maintainer) + for herd in new_herds: + herd = self.store_herd(*herds[herd]) + obj.herds.add(herd) - for maintainer in new_maintainers: - maintainer = maintainers[maintainer] - try: - maintainer = self.store_maintainer( - maintainer.name, maintainer.email + for maintainer in obj.maintainers.all(): + if maintainer.email in old_maintainers: + obj.maintainers.remove(maintainer) + + for maintainer in new_maintainers: + maintainer = maintainers[maintainer] + try: + maintainer = self.store_maintainer( + maintainer.name, maintainer.email ) - obj.maintainers.add(maintainer) - except ValidationError: - self.logger.error( - self.style.ERROR("Bad maintainer: '%s' '%s'" % \ + obj.maintainers.add(maintainer) + except ValidationError: + self.logger.error( + self.style.ERROR("Bad maintainer: '%s' '%s'" % \ (maintainer.name, maintainer.email)) ) obj.save() - return True def store_herd(self, name, email): if not name: @@ -134,15 +125,18 @@ class ScanMetadata(object): ) return maintainer - -def scan_metadata(packages=None, logger=None): +@commit_on_success +def scan_metadata(packages=None, category=None, logger=None): scan_handler = ScanMetadata(logger=logger) - if not packages: + + if category: + packages = Package.objects.filter(category=category) + elif not packages: packages = Package.objects.all() for pkg in packages: if isinstance(pkg, Package): - result = scan_handler.scan('%s/%s' % (pkg.category, pkg.name), pkg) + scan_handler.scan('%s/%s' % (pkg.category, pkg.name), pkg) else: - result = scan_handler.scan(pkg) - return result + scan_handler.scan(pkg) + diff --git a/euscanwww/djeuscan/processing/scan_portage.py b/euscanwww/djeuscan/processing/scan_portage.py index b8c9576..cbda779 100644 --- a/euscanwww/djeuscan/processing/scan_portage.py +++ b/euscanwww/djeuscan/processing/scan_portage.py @@ -302,4 +302,4 @@ def scan_portage(packages=None, category=None, no_log=False, purge_packages=Fals scan_handler.scan(pkg) logger.info('Done.') - return True +