From e535c204b021a3a87ae6b1d53248684786a603ae Mon Sep 17 00:00:00 2001 From: volpino Date: Mon, 28 May 2012 18:16:48 +0200 Subject: [PATCH 1/4] euscanwww: fixed homepage and category validation Accept multiple urls but skip invalid urls, added virtual as accepted category Signed-off-by: volpino --- euscanwww/djeuscan/models.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py index 6126110..727deef 100644 --- a/euscanwww/djeuscan/models.py +++ b/euscanwww/djeuscan/models.py @@ -1,11 +1,12 @@ from django.db import models from django.core.validators import RegexValidator, validate_email, URLValidator +from django.core.exceptions import ValidationError from djeuscan.managers import PackageManager, VersionLogManager, \ EuscanResultManager -validate_category = RegexValidator("^\w+?-\w+?$") +validate_category = RegexValidator("^(?:\w+?-\w+?)|virtual$") validate_name = RegexValidator("^\S+?$") validate_revision = RegexValidator("^r\d+?$") validate_url = URLValidator() @@ -56,7 +57,7 @@ class Package(models.Model): category = models.CharField(max_length=128, validators=[validate_category]) name = models.CharField(max_length=128, validators=[validate_name]) description = models.TextField(blank=True) - homepage = models.TextField(blank=True, validators=[validate_url]) + homepage = models.TextField(blank=True) herds = models.ManyToManyField(Herd, blank=True) maintainers = models.ManyToManyField(Maintainer, blank=True) @@ -89,6 +90,18 @@ class Package(models.Model): def save(self, *args, **kwargs): self.full_clean() + + # Clean urls, accept only real urls + urls = [] + for url in self.homepages: + try: + validate_url(url) + except ValidationError: + pass + else: + urls.append(url) + self.homepage = " ".join(urls) + super(Package, self).save(*args, **kwargs) @property From b46e9acc08c6cfceffc005f41697434dc0c316f9 Mon Sep 17 00:00:00 2001 From: volpino Date: Mon, 28 May 2012 21:16:38 +0200 Subject: [PATCH 2/4] euscan: Refactored all management commands Management commands are refactored to have the logic moved from the commands themselves to a separated function in order to make them easily convertible into Celery tasks Commands are renamed to be modules importable from other python scripts Signed-off-by: volpino --- .../management/commands/list-packages.py | 56 ---- .../management/commands/list_packages.py | 64 +++++ .../management/commands/regen-rrds.py | 23 -- .../management/commands/regen_rrds.py | 30 ++ .../{scan-metadata.py => scan_metadata.py} | 87 +++--- .../{scan-portage.py => scan_portage.py} | 220 +++++++------- .../{scan-upstream.py => scan_upstream.py} | 269 +++++++++--------- .../management/commands/update-counters.py | 199 ------------- .../management/commands/update_counters.py | 208 ++++++++++++++ .../djeuscan/management/commands/utils.py | 0 setup.py | 5 +- 11 files changed, 611 insertions(+), 550 deletions(-) delete mode 100644 euscanwww/djeuscan/management/commands/list-packages.py create mode 100644 euscanwww/djeuscan/management/commands/list_packages.py delete mode 100644 euscanwww/djeuscan/management/commands/regen-rrds.py create mode 100644 euscanwww/djeuscan/management/commands/regen_rrds.py rename euscanwww/djeuscan/management/commands/{scan-metadata.py => scan_metadata.py} (86%) rename euscanwww/djeuscan/management/commands/{scan-portage.py => scan_portage.py} (82%) rename euscanwww/djeuscan/management/commands/{scan-upstream.py => scan_upstream.py} (68%) delete mode 100644 euscanwww/djeuscan/management/commands/update-counters.py create mode 100644 euscanwww/djeuscan/management/commands/update_counters.py delete mode 100644 euscanwww/djeuscan/management/commands/utils.py diff --git a/euscanwww/djeuscan/management/commands/list-packages.py b/euscanwww/djeuscan/management/commands/list-packages.py deleted file mode 100644 index c1f0c38..0000000 --- a/euscanwww/djeuscan/management/commands/list-packages.py +++ /dev/null @@ -1,56 +0,0 @@ -from optparse import make_option - -from django.core.management.base import BaseCommand -from djeuscan.models import Package - - -class Command(BaseCommand): - _overlays = {} - help = 'List packages' - - option_list = BaseCommand.option_list + ( - make_option('--after', - action='store', - dest='after', - default=False, - help='After package'), - make_option('--before', - action='store', - dest='before', - default=False, - help='Before package'), - make_option('--limit', - action='store', - dest='limit', - default=False, - help='limit'), - ) - - def handle(self, *args, **options): - after = None - before = None - - if options['after']: - category, name = options['after'].split('/') - after = Package.objects.get(category=category, name=name) - - if options['before']: - category, name = options['before'].split('/') - before = Package.objects.get(category=category, name=name) - - packages = Package.objects - - if after or before: - if after: - packages = packages.filter(id__gte=after.id) - if before: - packages = packages.filter(id__lte=before.id) - else: - packages = packages.all() - - if options['limit']: - packages = packages[:int(options['limit'])] - - for pkg in packages: - self.stdout.write('%s/%s\n' % (pkg.category, pkg.name)) - self.stdout.close() diff --git a/euscanwww/djeuscan/management/commands/list_packages.py b/euscanwww/djeuscan/management/commands/list_packages.py new file mode 100644 index 0000000..23efe7a --- /dev/null +++ b/euscanwww/djeuscan/management/commands/list_packages.py @@ -0,0 +1,64 @@ +import sys +from optparse import make_option + +from django.core.management.base import BaseCommand +from djeuscan.models import Package + + +def list_packages(stdout=None, **options): + if stdout is None: + stdout = sys.stdout + + after = None + before = None + + if options['after']: + category, name = options['after'].split('/') + after = Package.objects.get(category=category, name=name) + + if options['before']: + category, name = options['before'].split('/') + before = Package.objects.get(category=category, name=name) + + packages = Package.objects + + if after or before: + if after: + packages = packages.filter(id__gte=after.id) + if before: + packages = packages.filter(id__lte=before.id) + else: + packages = packages.all() + + if options['limit']: + packages = packages[:int(options['limit'])] + + for pkg in packages: + stdout.write('%s/%s\n' % (pkg.category, pkg.name)) + stdout.close() + + +class Command(BaseCommand): + _overlays = {} + help = 'List packages' + + option_list = BaseCommand.option_list + ( + make_option('--after', + action='store', + dest='after', + default=False, + help='After package'), + make_option('--before', + action='store', + dest='before', + default=False, + help='Before package'), + make_option('--limit', + action='store', + dest='limit', + default=False, + help='limit'), + ) + + def handle(self, *args, **options): + list_packages(self.stdout, **options) diff --git a/euscanwww/djeuscan/management/commands/regen-rrds.py b/euscanwww/djeuscan/management/commands/regen-rrds.py deleted file mode 100644 index cbcbe0e..0000000 --- a/euscanwww/djeuscan/management/commands/regen-rrds.py +++ /dev/null @@ -1,23 +0,0 @@ -from django.core.management.base import BaseCommand -from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog -from djeuscan import charts - - -class Command(BaseCommand): - _overlays = {} - help = 'Regenerate rrd database' - - def handle(self, *args, **options): - for wlog in WorldLog.objects.all(): - charts.rrd_update('world', wlog.datetime, wlog) - - for clog in CategoryLog.objects.all(): - charts.rrd_update('category-%s' % clog.category, - clog.datetime, clog) - - for hlog in HerdLog.objects.all(): - charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog) - - for mlog in MaintainerLog.objects.all(): - charts.rrd_update('maintainer-%d' % mlog.maintainer.id, - mlog.datetime, mlog) diff --git a/euscanwww/djeuscan/management/commands/regen_rrds.py b/euscanwww/djeuscan/management/commands/regen_rrds.py new file mode 100644 index 0000000..1c041fa --- /dev/null +++ b/euscanwww/djeuscan/management/commands/regen_rrds.py @@ -0,0 +1,30 @@ +from django.core.management.base import BaseCommand +from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog +from djeuscan import charts + + +def regen_rrds(): + """ + Regenerates the rrd database + """ + for wlog in WorldLog.objects.all(): + charts.rrd_update('world', wlog.datetime, wlog) + + for clog in CategoryLog.objects.all(): + charts.rrd_update('category-%s' % clog.category, + clog.datetime, clog) + + for hlog in HerdLog.objects.all(): + charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog) + + for mlog in MaintainerLog.objects.all(): + charts.rrd_update('maintainer-%d' % mlog.maintainer.id, + mlog.datetime, mlog) + + +class Command(BaseCommand): + _overlays = {} + help = 'Regenerate rrd database' + + def handle(self, *args, **options): + regen_rrds() diff --git a/euscanwww/djeuscan/management/commands/scan-metadata.py b/euscanwww/djeuscan/management/commands/scan_metadata.py similarity index 86% rename from euscanwww/djeuscan/management/commands/scan-metadata.py rename to euscanwww/djeuscan/management/commands/scan_metadata.py index bcfbe49..153a18e 100644 --- a/euscanwww/djeuscan/management/commands/scan-metadata.py +++ b/euscanwww/djeuscan/management/commands/scan_metadata.py @@ -2,47 +2,22 @@ import sys from optparse import make_option -from django.db.transaction import commit_on_success -from django.core.management.base import BaseCommand -from djeuscan.models import Package, Herd, Maintainer - from gentoolkit.query import Query from gentoolkit.errors import GentoolkitFatalError +from django.db.transaction import commit_on_success +from django.core.management.base import BaseCommand +from django.core.management.color import color_style -class Command(BaseCommand): - _overlays = {} +from djeuscan.models import Package, Herd, Maintainer - option_list = BaseCommand.option_list + ( - make_option('--all', - action='store_true', - dest='all', - default=False, - help='Scan all packages'), - make_option('--quiet', - action='store_true', - dest='quiet', - default=False, - help='Be quiet'), - ) - args = '' - help = 'Scans metadata and fills database' + +class ScanMetadata(object): + def __init__(self, quiet): + self.quiet = quiet @commit_on_success - def handle(self, *args, **options): - self.options = options - - if options['all']: - for pkg in Package.objects.all(): - self.scan('%s/%s' % (pkg.category, pkg.name), pkg) - elif len(args): - for package in args: - self.scan(package) - else: - for package in sys.stdin.readlines(): - self.scan(package[:-1]) - - def scan(self, query=None, obj=None): + def run(self, query=None, obj=None): matches = Query(query).find( include_masked=True, in_installed=False, @@ -50,7 +25,7 @@ class Command(BaseCommand): if not matches: sys.stderr.write( - self.style.ERROR("Unknown package '%s'\n" % query) + color_style.ERROR("Unknown package '%s'\n" % query) ) return @@ -71,12 +46,12 @@ class Command(BaseCommand): obj.description = pkg.environment("DESCRIPTION") except GentoolkitFatalError, err: sys.stderr.write( - self.style.ERROR( + color_style.ERROR( "Gentoolkit fatal error: '%s'\n" % str(err) ) ) - if created and not self.options['quiet']: + if created and not self.quiet: sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name)) if pkg.metadata: @@ -127,7 +102,7 @@ class Command(BaseCommand): herd, created = Herd.objects.get_or_create(herd=name) - if created and not self.options['quiet']: + if created and not self.quiet: sys.stdout.write('+ [h] %s <%s>\n' % (name, email)) herd.email = email @@ -144,7 +119,7 @@ class Command(BaseCommand): maintainer, created = Maintainer.objects.get_or_create(email=email) if created: - if not self.options['quiet']: + if not self.quiet: sys.stdout.write( '+ [m] %s <%s>\n' % (name.encode('utf-8'), email) ) @@ -155,3 +130,37 @@ class Command(BaseCommand): maintainer.save() return maintainer + + +class Command(BaseCommand): + _overlays = {} + + option_list = BaseCommand.option_list + ( + make_option('--all', + action='store_true', + dest='all', + default=False, + help='Scan all packages'), + make_option('--quiet', + action='store_true', + dest='quiet', + default=False, + help='Be quiet'), + ) + args = '' + help = 'Scans metadata and fills database' + + def handle(self, *args, **options): + self.options = options + + scan_metadata = ScanMetadata(quiet=options["quiet"]) + + if options['all']: + for pkg in Package.objects.all(): + scan_metadata.run('%s/%s' % (pkg.category, pkg.name), pkg) + elif len(args) > 0: + for package in args: + scan_metadata.run(package) + else: + for package in sys.stdin.readlines(): + scan_metadata.run(package[:-1]) diff --git a/euscanwww/djeuscan/management/commands/scan-portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py similarity index 82% rename from euscanwww/djeuscan/management/commands/scan-portage.py rename to euscanwww/djeuscan/management/commands/scan_portage.py index 1d8b454..e38f081 100644 --- a/euscanwww/djeuscan/management/commands/scan-portage.py +++ b/euscanwww/djeuscan/management/commands/scan_portage.py @@ -8,49 +8,21 @@ from optparse import make_option from django.db.transaction import commit_on_success from django.core.management.base import BaseCommand +from django.core.management.color import color_style + from djeuscan.models import Package, Version, VersionLog -class Command(BaseCommand): - _overlays = {} +class ScanPortage(object): + def __init__(self, stdout=None, **options): + if stdout is None: + self.stdout = sys.stdout + else: + self.stdout = stdout - option_list = BaseCommand.option_list + ( - make_option('--all', - action='store_true', - dest='all', - default=False, - help='Scan all packages'), - make_option('--purge-packages', - action='store_true', - dest='purge-packages', - default=False, - help='Purge old packages'), - make_option('--purge-versions', - action='store_true', - dest='purge-versions', - default=False, - help='Purge old versions'), - make_option('--no-log', - action='store_true', - dest='no-log', - default=False, - help='Don\'t store logs'), - make_option('--prefetch', - action='store_true', - dest='prefetch', - default=False, - help=('Prefetch all versions and packages from DB to ' - 'speedup full scan process.')), - make_option('--quiet', - action='store_true', - dest='quiet', - default=False, - help='Be quiet'), - ) - args = '[package package ...]' - help = 'Scans portage tree and fills database' - - _cache = {'packages': {}, 'versions': {}} + self.options = options + self._cache = {'packages': {}, 'versions': {}} + self._overlays = None def cache_hash_package(self, category, name): return '%s/%s' % (category, name) @@ -84,38 +56,6 @@ class Command(BaseCommand): ) self._cache['versions'][key] = version - def handle(self, *args, **options): - self.options = options - - if not options['quiet']: - self.stdout.write('Scanning portage tree...\n') - - if options['prefetch']: - if not options['quiet']: - self.stdout.write('Prefetching objects...') - self.stdout.flush() - for package in Package.objects.all(): - self.cache_store_package(package) - for version in Version.objects.select_related('package').all(): - self.cache_store_version(version) - if not options['quiet']: - self.stdout.write('done\n') - - if options['all']: - self.scan() - elif len(args): - for package in args: - self.scan(package) - else: - for package in sys.stdin.readlines(): - self.scan(package[:-1]) - - if options['purge-versions']: - self.purge_versions(options) - - if not options['quiet']: - self.stdout.write('Done.\n') - def overlays(self): if self._overlays: return self._overlays @@ -143,7 +83,7 @@ class Command(BaseCommand): return self._overlays @commit_on_success - def scan(self, query=None): + def run(self, query=None): env = os.environ env['MY'] = "/-: []\n" @@ -177,7 +117,7 @@ class Command(BaseCommand): Package.objects.filter(name=query).delete() else: sys.stderr.write( - self.style.ERROR( + color_style.ERROR( "Unknown package '%s'\n" % query ) ) @@ -291,39 +231,113 @@ class Command(BaseCommand): if self.options['no-log']: return - entry = VersionLog.objects.create(package=obj.package, - action=VersionLog.VERSION_ADDED) - entry.slot = obj.slot - entry.revision = obj.revision - entry.version = obj.version - entry.overlay = obj.overlay + VersionLog.objects.create( + package=obj.package, + action=VersionLog.VERSION_ADDED, + slot=obj.slot, + revision=obj.revision, + version=obj.version, + overlay=obj.overlay + ) + + +@commit_on_success +def purge_versions(options): + # For each dead versions + for version in Version.objects.filter(packaged=True, alive=False): + if version.overlay == 'gentoo': + version.package.n_packaged -= 1 + else: + version.package.n_overlay -= 1 + version.package.n_versions -= 1 + version.package.save() + + if not options['quiet']: + sys.stdout.write('- [v] %s\n' % (version)) + + if options['no-log']: + continue + + entry = VersionLog.objects.create( + package=version.package, + action=VersionLog.VERSION_REMOVED + ) + entry.slot = version.slot + entry.revision = version.revision + entry.version = version.version + entry.overlay = version.overlay entry.save() - @commit_on_success - def purge_versions(self, options): - ' For each dead versions ' - for version in Version.objects.filter(packaged=True, alive=False): - if version.overlay == 'gentoo': - version.package.n_packaged -= 1 - else: - version.package.n_overlay -= 1 - version.package.n_versions -= 1 - version.package.save() + Version.objects.filter(packaged=True, alive=False).delete() - if not self.options['quiet']: - sys.stdout.write('- [v] %s\n' % (version)) - if self.options['no-log']: - continue +class Command(BaseCommand): + _overlays = {} - entry = VersionLog.objects.create( - package=version.package, - action=VersionLog.VERSION_REMOVED - ) - entry.slot = version.slot - entry.revision = version.revision - entry.version = version.version - entry.overlay = version.overlay - entry.save() + option_list = BaseCommand.option_list + ( + make_option('--all', + action='store_true', + dest='all', + default=False, + help='Scan all packages'), + make_option('--purge-packages', + action='store_true', + dest='purge-packages', + default=False, + help='Purge old packages'), + make_option('--purge-versions', + action='store_true', + dest='purge-versions', + default=False, + help='Purge old versions'), + make_option('--no-log', + action='store_true', + dest='no-log', + default=False, + help='Don\'t store logs'), + make_option('--prefetch', + action='store_true', + dest='prefetch', + default=False, + help=('Prefetch all versions and packages from DB to ' + 'speedup full scan process.')), + make_option('--quiet', + action='store_true', + dest='quiet', + default=False, + help='Be quiet'), + ) + args = '[package package ...]' + help = 'Scans portage tree and fills database' - Version.objects.filter(packaged=True, alive=False).delete() + def handle(self, *args, **options): + scan_portage = ScanPortage(stdout=self.stdout, **options) + + if not options['quiet']: + self.stdout.write('Scanning portage tree...\n') + + if options['prefetch']: + if not options['quiet']: + self.stdout.write('Prefetching objects...') + self.stdout.flush() + for package in Package.objects.all(): + scan_portage.cache_store_package(package) + for version in Version.objects.select_related('package').all(): + scan_portage.cache_store_version(version) + if not options['quiet']: + self.stdout.write('done\n') + + if options['all']: + scan_portage.run() + elif len(args): + for package in args: + scan_portage.run(package) + else: + for package in sys.stdin.readlines(): + scan_portage.run(package[:-1]) + + if options['purge-versions']: + purge_versions(options) + + if not options['quiet']: + self.stdout.write('Done.\n') diff --git a/euscanwww/djeuscan/management/commands/scan-upstream.py b/euscanwww/djeuscan/management/commands/scan_upstream.py similarity index 68% rename from euscanwww/djeuscan/management/commands/scan-upstream.py rename to euscanwww/djeuscan/management/commands/scan_upstream.py index dd00619..6dbdcfd 100644 --- a/euscanwww/djeuscan/management/commands/scan-upstream.py +++ b/euscanwww/djeuscan/management/commands/scan_upstream.py @@ -2,16 +2,150 @@ import subprocess import portage import sys import re - from StringIO import StringIO from optparse import make_option +from collections import defaultdict from django.utils import timezone from django.db.transaction import commit_on_success from django.core.management.base import BaseCommand + from djeuscan.models import Package, Version, EuscanResult, VersionLog +class ScanUpstream(object): + def __init__(self, options=None): + if options is None: + self.options = defaultdict(None) + else: + self.options = options + + def run(self, packages=None): + for package in packages: + cmd = ['euscan', package] + + fp = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output = StringIO(fp.communicate()[0]) + + self.parse_output(output) + + def store_result(self, package, log): + # Remove previous logs + EuscanResult.objects.filter(package=package).delete() + + obj = EuscanResult() + obj.package = package + obj.result = log + obj.datetime = timezone.now() + obj.save() + + def store_package(self, cpv): + cat, pkg, ver, rev = portage.catpkgsplit(cpv) + + obj, created = Package.objects.get_or_create(category=cat, name=pkg) + + if created and not self.options['quiet']: + sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg)) + + # Set all versions dead, then set found versions alive and + # delete old versions + Version.objects.filter(package=obj, packaged=False).update(alive=False) + + return obj + + def store_version(self, package, ver, url): + obj, created = Version.objects.get_or_create( + package=package, slot='', revision='r0', version=ver, overlay='' + ) + + obj.alive = True + obj.urls = url + obj.packaged = False + obj.save() + + # If it's not a new version, just update the object and continue + if not created: + return + + if not self.options['quiet']: + sys.stdout.write('+ [u] %s %s\n' % (obj, url)) + + entry = VersionLog.objects.create( + package=package, + action=VersionLog.VERSION_ADDED + ) + entry.slot = '' + entry.revision = 'r0' + entry.version = ver + entry.overlay = '' + entry.save() + + package.n_versions += 1 + package.save() + + @commit_on_success + def parse_output(self, output): + from portage.versions import _cp + + package_re = re.compile( + r'^ \* (?P' + _cp + ') \[(?P.*?)\]$' + ) + version_re = re.compile( + r'^Upstream Version: (?P.*?) (?P.*?)$' + ) + + package = None + log = "" + + while True: + line = output.readline() + if line == '': + break + match = package_re.match(line) + if match: + if package: + self.store_result(package, log) + + cpv = match.group('cpv') + package = self.store_package(cpv) + log = line + continue + + log += line + + match = version_re.match(line) + if match: + ver = match.group('ver') + url = match.group('url') + self.store_version(package, ver, url) + + if package: + self.store_result(package, log) + + +@commit_on_success +def purge_versions(options): + # For each dead versions + for version in Version.objects.filter(packaged=False, alive=False): + entry = VersionLog.objects.create( + package=version.package, + action=VersionLog.VERSION_REMOVED + ) + entry.slot = version.slot + entry.revision = version.revision + entry.version = version.version + entry.overlay = version.overlay + entry.save() + + version.package.n_versions -= 1 + version.package.save() + + if not options['quiet']: + sys.stdout.write('- [u] %s %s\n' % (version, version.urls)) + Version.objects.filter(packaged=False, alive=False).delete() + + class Command(BaseCommand): _overlays = {} @@ -41,10 +175,12 @@ class Command(BaseCommand): help = 'Scans metadata and fills database' def handle(self, *args, **options): + scan_upstream = ScanUpstream(options) + if options['feed']: - self.parse_output(options, sys.stdin) + scan_upstream.parse_output(options, sys.stdin) if options['purge-versions']: - self.purge_versions(options) + purge_versions(options) return if not options['quiet']: @@ -60,133 +196,10 @@ class Command(BaseCommand): else: packages = [package[:-1] for package in sys.stdin.readlines()] - self.scan(options, packages) + scan_upstream.run(packages) if options['purge-versions']: - self.purge_versions(options) + purge_versions(options) if not options['quiet']: self.stdout.write('Done.\n') - - def scan(self, options, packages=None): - for package in packages: - cmd = ['euscan', package] - - fp = subprocess.Popen(cmd, stdout=subprocess.PIPE) - output = StringIO(fp.communicate()[0]) - - self.parse_output(options, output) - - @commit_on_success - def parse_output(self, options, output): - from portage.versions import _cp - - package_re = re.compile( - r'^ \* (?P' + _cp + ') \[(?P.*?)\]$' - ) - version_re = re.compile( - r'^Upstream Version: (?P.*?) (?P.*?)$' - ) - - package = None - log = "" - - while True: - line = output.readline() - if line == '': - break - match = package_re.match(line) - if match: - if package: - self.store_result(options, package, log) - - cpv = match.group('cpv') - package = self.store_package(options, cpv) - log = line - continue - - log += line - - match = version_re.match(line) - if match: - ver = match.group('ver') - url = match.group('url') - self.store_version(options, package, ver, url) - - if package: - self.store_result(options, package, log) - - def store_result(self, options, package, log): - # Remove previous logs - EuscanResult.objects.filter(package=package).delete() - - obj = EuscanResult() - obj.package = package - obj.result = log - obj.datetime = timezone.now() - obj.save() - - def store_package(self, options, cpv): - cat, pkg, ver, rev = portage.catpkgsplit(cpv) - - obj, created = Package.objects.get_or_create(category=cat, name=pkg) - - if created and not options['quiet']: - sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg)) - - # Set all versions dead, then set found versions alive and - # delete old versions - Version.objects.filter(package=obj, packaged=False).update(alive=False) - - return obj - - def store_version(self, options, package, ver, url): - obj, created = Version.objects.get_or_create( - package=package, slot='', revision='r0', version=ver, overlay='' - ) - - obj.alive = True - obj.urls = url - obj.packaged = False - obj.save() - - ''' If it's not a new version, just update the object and continue ''' - if not created: - return - - if not options['quiet']: - sys.stdout.write('+ [u] %s %s\n' % (obj, url)) - - entry = VersionLog.objects.create( - package=package, - action=VersionLog.VERSION_ADDED - ) - entry.slot = '' - entry.revision = 'r0' - entry.version = ver - entry.overlay = '' - entry.save() - - package.n_versions += 1 - package.save() - - @commit_on_success - def purge_versions(self, options): - ' For each dead versions ' - for version in Version.objects.filter(packaged=False, alive=False): - entry = VersionLog.objects.create( - package=version.package, - action=VersionLog.VERSION_REMOVED - ) - entry.slot = version.slot - entry.revision = version.revision - entry.version = version.version - entry.overlay = version.overlay - entry.save() - - version.package.n_versions -= 1 - version.package.save() - - if not options['quiet']: - sys.stdout.write('- [u] %s %s\n' % (version, version.urls)) - Version.objects.filter(packaged=False, alive=False).delete() diff --git a/euscanwww/djeuscan/management/commands/update-counters.py b/euscanwww/djeuscan/management/commands/update-counters.py deleted file mode 100644 index de3ea03..0000000 --- a/euscanwww/djeuscan/management/commands/update-counters.py +++ /dev/null @@ -1,199 +0,0 @@ -from optparse import make_option - -from django.db.transaction import commit_on_success -from django.core.management.base import BaseCommand -from django.utils import timezone - -from djeuscan.models import Package, Herd, Maintainer, Version -from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog -from djeuscan import charts - -from distutils.version import StrictVersion, LooseVersion - - -def compare_versions(version1, version2): - try: - return cmp(StrictVersion(version1), StrictVersion(version2)) - # in case of abnormal version number, fall back to LooseVersion - except ValueError: - return cmp(LooseVersion(version1), LooseVersion(version2)) - - -class Command(BaseCommand): - _overlays = {} - help = 'Update counters' - - option_list = BaseCommand.option_list + ( - make_option('--quiet', - action='store_true', - dest='quiet', - default=False, - help='Be quiet'), - make_option('--fast', - action='store_true', - dest='fast', - default=False, - help='Skip sanity checks'), - make_option('--nolog', - action='store_true', - dest='nolog', - default=False, - help='Skip logs'), - - ) - - @commit_on_success - def handle(self, *args, **options): - now = timezone.now() - - categories = {} - herds = {} - maintainers = {} - - wlog = None - - if not options['nolog']: - wlog = WorldLog() - wlog.datetime = now - - for cat in Package.objects.values('category').distinct(): - clog = CategoryLog() - clog.datetime = now - clog.category = cat['category'] - categories[clog.category] = clog - - for herd in Herd.objects.all(): - hlog = HerdLog() - hlog.datetime = now - hlog.herd = herd - herds[herd.id] = hlog - - for maintainer in Maintainer.objects.all(): - mlog = MaintainerLog() - mlog.datetime = now - mlog.maintainer = maintainer - maintainers[maintainer.id] = mlog - - package_queryset = Package.objects.all() - - n_versions = {} - n_packaged = {} - n_overlay = {} - - last_versions_gentoo = {} - last_versions_overlay = {} - last_versions_upstream = {} - - def add_safe(storage, key): - if key not in storage: - storage[key] = 1 - else: - storage[key] += 1 - - def add_last_ver(storage, version): - key = version['package_id'] - if key not in storage: - storage[key] = version - return - if version['version'].startswith('9999'): - return - if compare_versions(storage[key]['version'], - version['version']) < 0: - storage[key] = version - - if not options['fast']: - attrs = ['id', 'version', 'overlay', 'packaged', 'package_id'] - for version in Version.objects.all().values(*attrs): - overlay, packaged = version['overlay'], version['packaged'] - package_id = version['package_id'] - - add_safe(n_versions, package_id) - - if not packaged: - add_last_ver(last_versions_upstream, version) - continue - if overlay == 'gentoo': - add_safe(n_packaged, package_id) - add_last_ver(last_versions_gentoo, version) - else: - add_safe(n_overlay, package_id) - add_last_ver(last_versions_overlay, version) - - for package in package_queryset.select_related('herds', 'maintainers'): - if not options['fast']: - package.n_versions = n_versions.get(package.id, 0) - package.n_packaged = n_packaged.get(package.id, 0) - package.n_overlay = n_overlay.get(package.id, 0) - - default = {'id': None} - package.last_version_gentoo_id = last_versions_gentoo.get( - package.id, default - )['id'] - package.last_version_overlay_id = last_versions_overlay.get( - package.id, default - )['id'] - package.last_version_upstream_id = last_versions_upstream.get( - package.id, default - )['id'] - - package.save() - - n_packages_gentoo = int(package.n_packaged == package.n_versions) - n_packages_overlay = int(package.n_overlay and package.n_packaged \ - + package.n_overlay == package.n_versions) - n_packages_outdated = int(package.n_packaged + package.n_overlay \ - < package.n_versions) - - def update_row(storage, key): - storage[key].n_packages_gentoo += n_packages_gentoo - storage[key].n_packages_overlay += n_packages_overlay - storage[key].n_packages_outdated += n_packages_outdated - - storage[key].n_versions_gentoo += package.n_packaged - storage[key].n_versions_overlay += package.n_overlay - storage[key].n_versions_upstream += package.n_versions - \ - package.n_packaged - \ - package.n_overlay - - def update_log(storage, qs): - for row in qs: - update_row(storage, row['id']) - - if not options['nolog']: - update_log(herds, package.herds.all().values('id')) - update_log(maintainers, package.maintainers.all().values('id')) - update_row(categories, package.category) - - wlog.n_packages_gentoo += n_packages_gentoo - wlog.n_packages_overlay += n_packages_overlay - wlog.n_packages_outdated += n_packages_outdated - - wlog.n_versions_gentoo += package.n_packaged - wlog.n_versions_overlay += package.n_overlay - wlog.n_versions_upstream += package.n_versions - \ - package.n_packaged - \ - package.n_overlay - - if options['nolog']: - return - - for clog in categories.values(): - if not options['quiet']: - self.stdout.write('+ [cl] %s\n' % clog) - charts.rrd_update('category-%s' % clog.category, now, clog) - clog.save() - - for hlog in herds.values(): - if not options['quiet']: - self.stdout.write('+ [hl] %s\n' % hlog) - charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog) - hlog.save() - - for mlog in maintainers.values(): - if not options['quiet']: - self.stdout.write('+ [ml] %s\n' % mlog) - charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog) - mlog.save() - - charts.rrd_update('world', now, wlog) - wlog.save() diff --git a/euscanwww/djeuscan/management/commands/update_counters.py b/euscanwww/djeuscan/management/commands/update_counters.py new file mode 100644 index 0000000..0b67a39 --- /dev/null +++ b/euscanwww/djeuscan/management/commands/update_counters.py @@ -0,0 +1,208 @@ +import sys +from optparse import make_option + +from django.db.transaction import commit_on_success +from django.core.management.base import BaseCommand +from django.utils import timezone + +from djeuscan.models import Package, Herd, Maintainer, Version +from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog +from djeuscan import charts + +from distutils.version import StrictVersion, LooseVersion + + +def compare_versions(version1, version2): + try: + return cmp(StrictVersion(version1), StrictVersion(version2)) + # in case of abnormal version number, fall back to LooseVersion + except ValueError: + return cmp(LooseVersion(version1), LooseVersion(version2)) + + +def add_safe(storage, key): + if key not in storage: + storage[key] = 1 + else: + storage[key] += 1 + + +def add_last_ver(storage, version): + key = version['package_id'] + if key not in storage: + storage[key] = version + return + if version['version'].startswith('9999'): + return + if compare_versions(storage[key]['version'], + version['version']) < 0: + storage[key] = version + + +@commit_on_success +def update_counters(stdout=None, **options): + if stdout is None: + stdout = sys.stdout + + now = timezone.now() + + categories = {} + herds = {} + maintainers = {} + + wlog = None + + if not options['nolog']: + wlog = WorldLog() + wlog.datetime = now + + for cat in Package.objects.values('category').distinct(): + clog = CategoryLog() + clog.datetime = now + clog.category = cat['category'] + categories[clog.category] = clog + + for herd in Herd.objects.all(): + hlog = HerdLog() + hlog.datetime = now + hlog.herd = herd + herds[herd.id] = hlog + + for maintainer in Maintainer.objects.all(): + mlog = MaintainerLog() + mlog.datetime = now + mlog.maintainer = maintainer + maintainers[maintainer.id] = mlog + + package_queryset = Package.objects.all() + + n_versions = {} + n_packaged = {} + n_overlay = {} + + last_versions_gentoo = {} + last_versions_overlay = {} + last_versions_upstream = {} + + if not options['fast']: + attrs = ['id', 'version', 'overlay', 'packaged', 'package_id'] + for version in Version.objects.all().values(*attrs): + overlay, packaged = version['overlay'], version['packaged'] + package_id = version['package_id'] + + add_safe(n_versions, package_id) + + if not packaged: + add_last_ver(last_versions_upstream, version) + continue + if overlay == 'gentoo': + add_safe(n_packaged, package_id) + add_last_ver(last_versions_gentoo, version) + else: + add_safe(n_overlay, package_id) + add_last_ver(last_versions_overlay, version) + + for package in package_queryset.select_related('herds', 'maintainers'): + if not options['fast']: + package.n_versions = n_versions.get(package.id, 0) + package.n_packaged = n_packaged.get(package.id, 0) + package.n_overlay = n_overlay.get(package.id, 0) + + default = {'id': None} + package.last_version_gentoo_id = last_versions_gentoo.get( + package.id, default + )['id'] + package.last_version_overlay_id = last_versions_overlay.get( + package.id, default + )['id'] + package.last_version_upstream_id = last_versions_upstream.get( + package.id, default + )['id'] + + package.save() + + n_packages_gentoo = int(package.n_packaged == package.n_versions) + n_packages_overlay = int(package.n_overlay and package.n_packaged \ + + package.n_overlay == package.n_versions) + n_packages_outdated = int(package.n_packaged + package.n_overlay \ + < package.n_versions) + + def update_row(storage, key): + storage[key].n_packages_gentoo += n_packages_gentoo + storage[key].n_packages_overlay += n_packages_overlay + storage[key].n_packages_outdated += n_packages_outdated + + storage[key].n_versions_gentoo += package.n_packaged + storage[key].n_versions_overlay += package.n_overlay + storage[key].n_versions_upstream += package.n_versions - \ + package.n_packaged - \ + package.n_overlay + + def update_log(storage, qs): + for row in qs: + update_row(storage, row['id']) + + if not options['nolog']: + update_log(herds, package.herds.all().values('id')) + update_log(maintainers, package.maintainers.all().values('id')) + update_row(categories, package.category) + + wlog.n_packages_gentoo += n_packages_gentoo + wlog.n_packages_overlay += n_packages_overlay + wlog.n_packages_outdated += n_packages_outdated + + wlog.n_versions_gentoo += package.n_packaged + wlog.n_versions_overlay += package.n_overlay + wlog.n_versions_upstream += package.n_versions - \ + package.n_packaged - \ + package.n_overlay + + if options['nolog']: + return + + for clog in categories.values(): + if not options['quiet']: + stdout.write('+ [cl] %s\n' % clog) + charts.rrd_update('category-%s' % clog.category, now, clog) + clog.save() + + for hlog in herds.values(): + if not options['quiet']: + stdout.write('+ [hl] %s\n' % hlog) + charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog) + hlog.save() + + for mlog in maintainers.values(): + if not options['quiet']: + stdout.write('+ [ml] %s\n' % mlog) + charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog) + mlog.save() + + charts.rrd_update('world', now, wlog) + wlog.save() + + +class Command(BaseCommand): + _overlays = {} + help = 'Update counters' + + option_list = BaseCommand.option_list + ( + make_option('--quiet', + action='store_true', + dest='quiet', + default=False, + help='Be quiet'), + make_option('--fast', + action='store_true', + dest='fast', + default=False, + help='Skip sanity checks'), + make_option('--nolog', + action='store_true', + dest='nolog', + default=False, + help='Skip logs'), + ) + + def handle(self, *args, **options): + update_counters(stdout=self.stdout, **options) diff --git a/euscanwww/djeuscan/management/commands/utils.py b/euscanwww/djeuscan/management/commands/utils.py deleted file mode 100644 index e69de29..0000000 diff --git a/setup.py b/setup.py index baee02f..240f459 100755 --- a/setup.py +++ b/setup.py @@ -87,8 +87,9 @@ setup( 'https://github.com/iksaif/euscan/tarball/' + ('master' if __version__ == '9999' else ('euscan-%s' % __version__)) ), - install_requires=['Django==1.4', 'django-annoying', 'South', - 'django-piston', 'BeautifulSoup', "matplotlib"], + install_requires=['Django==1.4', 'django-annoying==0..7.6', 'South==0.7.4', + 'django-piston==0.2.3', 'BeautifulSoup==3.2.1', + 'matplotlib==1.1.0', 'django-celery==2.5.5'], package_dir={'': 'pym'}, packages=packages, package_data={}, From fce8ef347b9b6c10669bce43d22bfc23d2609a9a Mon Sep 17 00:00:00 2001 From: volpino Date: Mon, 28 May 2012 22:57:33 +0200 Subject: [PATCH 3/4] euscanwww: management commands are validators aware Management commands should create objects that don't raise ValidationError Signed-off-by: volpino --- .../management/commands/scan_metadata.py | 16 ++++---- .../management/commands/scan_portage.py | 24 ++++++------ .../management/commands/scan_upstream.py | 39 +++++++++---------- 3 files changed, 39 insertions(+), 40 deletions(-) diff --git a/euscanwww/djeuscan/management/commands/scan_metadata.py b/euscanwww/djeuscan/management/commands/scan_metadata.py index 153a18e..c8e926a 100644 --- a/euscanwww/djeuscan/management/commands/scan_metadata.py +++ b/euscanwww/djeuscan/management/commands/scan_metadata.py @@ -100,7 +100,10 @@ class ScanMetadata(object): name = '{nil}' name = name.strip("\r").strip("\n").strip("\t").strip() - herd, created = Herd.objects.get_or_create(herd=name) + herd, created = Herd.objects.get_or_create( + herd=name, + defaults={"email": email} + ) if created and not self.quiet: sys.stdout.write('+ [h] %s <%s>\n' % (name, email)) @@ -116,19 +119,16 @@ class ScanMetadata(object): if not name: name = '{nil}' - maintainer, created = Maintainer.objects.get_or_create(email=email) + maintainer, created = Maintainer.objects.get_or_create( + email=email, + defaults={"name": name} + ) if created: if not self.quiet: sys.stdout.write( '+ [m] %s <%s>\n' % (name.encode('utf-8'), email) ) - - if not maintainer.name or \ - name not in [maintainer.name, email, '{nil}']: - maintainer.name = name - maintainer.save() - return maintainer diff --git a/euscanwww/djeuscan/management/commands/scan_portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py index e38f081..58997ad 100644 --- a/euscanwww/djeuscan/management/commands/scan_portage.py +++ b/euscanwww/djeuscan/management/commands/scan_portage.py @@ -203,12 +203,13 @@ class ScanPortage(object): obj, created = Version.objects.get_or_create( package=package, slot=slot, revision=rev, version=ver, - overlay=overlay + overlay=overlay, + defaults={"alive": True, "packaged": True} ) - - obj.alive = True - obj.packaged = True - obj.save() + if not created: + obj.alive = True + obj.packaged = True + obj.save() if created: self.cache_store_version(obj) @@ -258,15 +259,14 @@ def purge_versions(options): if options['no-log']: continue - entry = VersionLog.objects.create( + VersionLog.objects.create( package=version.package, - action=VersionLog.VERSION_REMOVED + action=VersionLog.VERSION_REMOVED, + slot=version.slot, + revision=version.revision, + version=version.version, + overlay=version.overlay ) - entry.slot = version.slot - entry.revision = version.revision - entry.version = version.version - entry.overlay = version.overlay - entry.save() Version.objects.filter(packaged=True, alive=False).delete() diff --git a/euscanwww/djeuscan/management/commands/scan_upstream.py b/euscanwww/djeuscan/management/commands/scan_upstream.py index 6dbdcfd..f6e78c3 100644 --- a/euscanwww/djeuscan/management/commands/scan_upstream.py +++ b/euscanwww/djeuscan/management/commands/scan_upstream.py @@ -56,13 +56,14 @@ class ScanUpstream(object): def store_version(self, package, ver, url): obj, created = Version.objects.get_or_create( - package=package, slot='', revision='r0', version=ver, overlay='' + package=package, slot='', revision='r0', version=ver, overlay='', + defaults={"alive": True, "urls": url, "packaged": True} ) - - obj.alive = True - obj.urls = url - obj.packaged = False - obj.save() + if not created: + obj.alive = True + obj.urls = url + obj.packaged = False + obj.save() # If it's not a new version, just update the object and continue if not created: @@ -71,15 +72,14 @@ class ScanUpstream(object): if not self.options['quiet']: sys.stdout.write('+ [u] %s %s\n' % (obj, url)) - entry = VersionLog.objects.create( + VersionLog.objects.create( package=package, - action=VersionLog.VERSION_ADDED + action=VersionLog.VERSION_ADDED, + slot='', + revision='r0', + version=ver, + overlay='' ) - entry.slot = '' - entry.revision = 'r0' - entry.version = ver - entry.overlay = '' - entry.save() package.n_versions += 1 package.save() @@ -128,15 +128,14 @@ class ScanUpstream(object): def purge_versions(options): # For each dead versions for version in Version.objects.filter(packaged=False, alive=False): - entry = VersionLog.objects.create( + VersionLog.objects.create( package=version.package, - action=VersionLog.VERSION_REMOVED + action=VersionLog.VERSION_REMOVED, + slot=version.slot, + revision=version.revision, + version=version.version, + overlay=version.overlay ) - entry.slot = version.slot - entry.revision = version.revision - entry.version = version.version - entry.overlay = version.overlay - entry.save() version.package.n_versions -= 1 version.package.save() From c36f625a54e7a99a49462ce64df9d996a18f741f Mon Sep 17 00:00:00 2001 From: volpino Date: Mon, 28 May 2012 23:24:09 +0200 Subject: [PATCH 4/4] euscanwww: fixing colored output Signed-off-by: volpino --- euscanwww/djeuscan/management/commands/scan_metadata.py | 5 +++-- euscanwww/djeuscan/management/commands/scan_portage.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/euscanwww/djeuscan/management/commands/scan_metadata.py b/euscanwww/djeuscan/management/commands/scan_metadata.py index c8e926a..8590412 100644 --- a/euscanwww/djeuscan/management/commands/scan_metadata.py +++ b/euscanwww/djeuscan/management/commands/scan_metadata.py @@ -15,6 +15,7 @@ from djeuscan.models import Package, Herd, Maintainer class ScanMetadata(object): def __init__(self, quiet): self.quiet = quiet + self.style = color_style() @commit_on_success def run(self, query=None, obj=None): @@ -25,7 +26,7 @@ class ScanMetadata(object): if not matches: sys.stderr.write( - color_style.ERROR("Unknown package '%s'\n" % query) + self.style.ERROR("Unknown package '%s'\n" % query) ) return @@ -46,7 +47,7 @@ class ScanMetadata(object): obj.description = pkg.environment("DESCRIPTION") except GentoolkitFatalError, err: sys.stderr.write( - color_style.ERROR( + self.style.ERROR( "Gentoolkit fatal error: '%s'\n" % str(err) ) ) diff --git a/euscanwww/djeuscan/management/commands/scan_portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py index 58997ad..4227501 100644 --- a/euscanwww/djeuscan/management/commands/scan_portage.py +++ b/euscanwww/djeuscan/management/commands/scan_portage.py @@ -21,6 +21,7 @@ class ScanPortage(object): self.stdout = stdout self.options = options + self.style = color_style() self._cache = {'packages': {}, 'versions': {}} self._overlays = None @@ -117,7 +118,7 @@ class ScanPortage(object): Package.objects.filter(name=query).delete() else: sys.stderr.write( - color_style.ERROR( + self.style.ERROR( "Unknown package '%s'\n" % query ) )