From 3437cf5e4b0a57c55d8ca94db0c6faa246dc2639 Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Mon, 28 May 2012 07:02:41 +0200 Subject: [PATCH 1/3] euscan: cleanups Signed-off-by: Corentin Chary --- pym/euscan/scan.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py index 0b8f4ad..6dde062 100644 --- a/pym/euscan/scan.py +++ b/pym/euscan/scan.py @@ -7,7 +7,6 @@ from portage.dbapi import porttree import gentoolkit.pprinter as pp from gentoolkit.query import Query from gentoolkit.package import Package -from gentoolkit.eclean.search import (port_settings) from euscan import CONFIG, BLACKLIST_PACKAGES from euscan import handlers, helpers, output @@ -92,10 +91,15 @@ def scan_upstream_urls(cpv, urls, on_progress): # gentoolkit stores PORTDB, so even if we modify it to add an overlay # it will still use the old dbapi def reload_gentoolkit(): + from gentoolkit import dbapi import gentoolkit.package import gentoolkit.query PORTDB = portage.db[portage.root]["porttree"].dbapi + dbapi.PORTDB = PORTDB + + if hasattr(dbapi, 'PORTDB'): + dbapi.PORTDB = PORTDB if hasattr(gentoolkit.package, 'PORTDB'): gentoolkit.package.PORTDB = PORTDB if hasattr(gentoolkit.query, 'PORTDB'): @@ -179,10 +183,10 @@ def scan_upstream(query, on_progress=None): cpv = pkg.cpv metadata = { - "EAPI": port_settings["EAPI"], + "EAPI": portage.settings["EAPI"], "SRC_URI": pkg.environment("SRC_URI", False), } - use = frozenset(port_settings["PORTAGE_USE"].split()) + use = frozenset(portage.settings["PORTAGE_USE"].split()) try: alist = porttree._parse_uri_map(cpv, metadata, use=use) aalist = porttree._parse_uri_map(cpv, metadata) From 83f5df4daf6fc6d074b9f0afb51bb3bbaad9a33c Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Tue, 29 May 2012 15:29:03 +0200 Subject: [PATCH 2/3] euscanwww: fix the scan process Signed-off-by: Corentin Chary --- .../djeuscan/management/commands/scan_metadata.py | 14 ++++++++++---- .../djeuscan/management/commands/scan_portage.py | 1 - .../djeuscan/management/commands/scan_upstream.py | 5 ++++- euscanwww/djeuscan/models.py | 11 ++++++----- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/euscanwww/djeuscan/management/commands/scan_metadata.py b/euscanwww/djeuscan/management/commands/scan_metadata.py index 8590412..d9ec08c 100644 --- a/euscanwww/djeuscan/management/commands/scan_metadata.py +++ b/euscanwww/djeuscan/management/commands/scan_metadata.py @@ -8,6 +8,7 @@ from gentoolkit.errors import GentoolkitFatalError from django.db.transaction import commit_on_success from django.core.management.base import BaseCommand from django.core.management.color import color_style +from django.core.exceptions import ValidationError from djeuscan.models import Package, Herd, Maintainer @@ -89,10 +90,15 @@ class ScanMetadata(object): for maintainer in new_maintainers: maintainer = maintainers[maintainer] - maintainer = self.store_maintainer( - maintainer.name, maintainer.email - ) - obj.maintainers.add(maintainer) + try: + maintainer = self.store_maintainer( + maintainer.name, maintainer.email + ) + obj.maintainers.add(maintainer) + except ValidationError: + sys.stderr.write( + self.style.ERROR("Bad maintainer: '%s' '%s'\n" % (maintainer.name, maintainer.email)) + ) obj.save() diff --git a/euscanwww/djeuscan/management/commands/scan_portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py index 4227501..86481e1 100644 --- a/euscanwww/djeuscan/management/commands/scan_portage.py +++ b/euscanwww/djeuscan/management/commands/scan_portage.py @@ -242,7 +242,6 @@ class ScanPortage(object): overlay=obj.overlay ) - @commit_on_success def purge_versions(options): # For each dead versions diff --git a/euscanwww/djeuscan/management/commands/scan_upstream.py b/euscanwww/djeuscan/management/commands/scan_upstream.py index f6e78c3..c5e95b0 100644 --- a/euscanwww/djeuscan/management/commands/scan_upstream.py +++ b/euscanwww/djeuscan/management/commands/scan_upstream.py @@ -88,6 +88,9 @@ class ScanUpstream(object): def parse_output(self, output): from portage.versions import _cp + if type(_cp) == dict: + _cp = _cp["dots_allowed_in_PN"] + package_re = re.compile( r'^ \* (?P' + _cp + ') \[(?P.*?)\]$' ) @@ -177,7 +180,7 @@ class Command(BaseCommand): scan_upstream = ScanUpstream(options) if options['feed']: - scan_upstream.parse_output(options, sys.stdin) + scan_upstream.parse_output(sys.stdin) if options['purge-versions']: purge_versions(options) return diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py index 727deef..9445af4 100644 --- a/euscanwww/djeuscan/models.py +++ b/euscanwww/djeuscan/models.py @@ -115,11 +115,12 @@ class Version(models.Model): """ package = models.ForeignKey(Package) - slot = models.CharField(max_length=128) + slot = models.CharField(max_length=128, blank=True, default='') revision = models.CharField(max_length=128) version = models.CharField(max_length=128) packaged = models.BooleanField() - overlay = models.CharField(max_length=128, default='gentoo', db_index=True, + overlay = models.CharField(max_length=128, blank=True, + default='gentoo', db_index=True, validators=[validate_name]) urls = models.TextField(blank=True) alive = models.BooleanField(default=True, db_index=True) @@ -148,12 +149,12 @@ class VersionLog(models.Model): package = models.ForeignKey(Package) datetime = models.DateTimeField(auto_now_add=True) - slot = models.CharField(max_length=128) + slot = models.CharField(max_length=128, blank=True, default='') revision = models.CharField(max_length=128) version = models.CharField(max_length=128) packaged = models.BooleanField() - overlay = models.CharField(max_length=128, default='gentoo', - validators=[validate_name]) + overlay = models.CharField(max_length=128, blank=True, + default='gentoo', validators=[validate_name]) action = models.IntegerField(choices=VERSION_ACTIONS) objects = VersionLogManager() From 5b1884ead76fb97ec85eb61511dd6b1f0934dcb7 Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Mon, 4 Jun 2012 10:28:06 +0200 Subject: [PATCH 3/3] djeuscan: fix alive / packaged updates Signed-off-by: Corentin Chary --- euscanwww/djeuscan/management/commands/scan_portage.py | 8 ++++---- euscanwww/djeuscan/management/commands/scan_upstream.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/euscanwww/djeuscan/management/commands/scan_portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py index 86481e1..d5837ec 100644 --- a/euscanwww/djeuscan/management/commands/scan_portage.py +++ b/euscanwww/djeuscan/management/commands/scan_portage.py @@ -207,10 +207,10 @@ class ScanPortage(object): overlay=overlay, defaults={"alive": True, "packaged": True} ) - if not created: - obj.alive = True - obj.packaged = True - obj.save() + if not created: # Created objects have defaults values + obj.alive = True + obj.packaged = True + obj.save() if created: self.cache_store_version(obj) diff --git a/euscanwww/djeuscan/management/commands/scan_upstream.py b/euscanwww/djeuscan/management/commands/scan_upstream.py index c5e95b0..8014b7c 100644 --- a/euscanwww/djeuscan/management/commands/scan_upstream.py +++ b/euscanwww/djeuscan/management/commands/scan_upstream.py @@ -57,7 +57,7 @@ class ScanUpstream(object): def store_version(self, package, ver, url): obj, created = Version.objects.get_or_create( package=package, slot='', revision='r0', version=ver, overlay='', - defaults={"alive": True, "urls": url, "packaged": True} + defaults={"alive": True, "urls": url, "packaged": False} ) if not created: obj.alive = True