diff --git a/bin/euscan b/bin/euscan index 907902a..963193f 100755 --- a/bin/euscan +++ b/bin/euscan @@ -21,8 +21,8 @@ __description__ = "A tool to detect new upstream releases." import sys import os import getopt -import errno -import httplib +from errno import EINTR, EINVAL +from httplib import HTTPConnection from portage.output import white, yellow, turquoise, green from portage.exception import AmbiguousPackageName @@ -37,6 +37,8 @@ from euscan.out import progress_bar # Globals +isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() + def exit_helper(status): if CONFIG["format"]: @@ -52,7 +54,7 @@ def setup_signals(): signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) print() - exit_helper(errno.EINTR) + exit_helper(EINTR) signal.signal(signal.SIGINT, exithandler) signal.signal(signal.SIGTERM, exithandler) @@ -121,6 +123,8 @@ def print_usage(_error=None, help=None): print(yellow(" -f, --format=") + " - define the output " + yellow("") + " (available: json)", file=out) + print(yellow(" -p, --progress") + + " - display a progress bar", file=out) print(file=out) if _error in ('packages',) or help: @@ -172,6 +176,8 @@ def parse_args(): CONFIG['format'] = a CONFIG['nocolor'] = True pp.output.nocolor() + elif o in ("-p", "--progress"): + CONFIG['progress'] = isatty else: return_code = False @@ -179,7 +185,7 @@ def parse_args(): # here are the different allowed command line options (getopt args) getopt_options = {'short': {}, 'long': {}} - getopt_options['short']['global'] = "hVCqv1bf:" + getopt_options['short']['global'] = "hVCqv1bf:p" getopt_options['long']['global'] = [ "help", "version", "nocolor", "quiet", "verbose", "oneshot", "brute-force=", "format=" @@ -207,7 +213,7 @@ def parse_args(): def main(): """Parse command line and execute all actions.""" CONFIG['nocolor'] = ( - port_settings["NOCOLOR"] in ('yes', 'true') or not sys.stdout.isatty() + port_settings["NOCOLOR"] in ('yes', 'true') or not isatty ) if CONFIG['nocolor']: pp.output.nocolor() @@ -230,19 +236,23 @@ def main(): else: print_usage(e.value) - exit_helper(errno.EINVAL) + exit_helper(EINVAL) if CONFIG['verbose'] > 2: - httplib.HTTPConnection.debuglevel = 1 + HTTPConnection.debuglevel = 1 - isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() + if not CONFIG["format"]: + CONFIG["progress"] = False + + on_progress = None + if CONFIG['progress']: + on_progress_gen = progress_bar() + on_progress = on_progress_gen.next() + on_progress(maxval=len(queries) * 100, increment=0) for query in queries: - on_progress = None - if (CONFIG['format'] or CONFIG['quiet']) and isatty: - print("%s:" % query, file=sys.stderr) - on_progress_gen = progress_bar() - on_progress = on_progress_gen.next() + if CONFIG["progress"]: + on_progress(increment=10, label=query) ret = [] @@ -271,26 +281,15 @@ def main(): output.eerror('%s: %s' % (query, str(err))) exit_helper(1) - if not CONFIG['quiet'] and not CONFIG['format']: - print() + if not ret and not CONFIG['quiet']: + output.ewarn( + "Didn't find any new version, check package's homepage " + + "for more informations" + ) - if (CONFIG['format'] or CONFIG['quiet']) and isatty: - on_progress_gen.next() - print("\n", file=sys.stderr) - - if ret is not None: - if len(ret) > 0: - for cp, url, version, handler, confidence in ret: - output.result(cp, version, url, handler, confidence) - - if (CONFIG['format'] or CONFIG['quiet']) and isatty: - print("\n", file=sys.stderr) - - elif not CONFIG['quiet']: - output.ewarn( - "Didn't find any new version, check package's homepage " + - "for more informations" - ) + if CONFIG['progress']: + on_progress_gen.next() + print("\n", file=sys.stderr) output.set_query(None) diff --git a/euscanwww/README.Celery b/euscanwww/README.Celery new file mode 100644 index 0000000..5159e32 --- /dev/null +++ b/euscanwww/README.Celery @@ -0,0 +1,40 @@ +Requirements +============ + +Installing requirements with setup.py is the easiest way, just run:: + + python setup.py develop + +or:: + + python setup.py install # to install euscan and requirements + +If you prefer to use portage just install dev-python/django-celery-2.5.5 + +There's the need of having a broker for tasks. The default and reccommended +broker is RabbitMQ. +Install with:: +emerge rabbitmq-server + + +Running Celery +============== + +You'll need: +* celeryd (celery daemon for running tasks):: + + python manage.py celeryd -E -l INFO + +* celerycam (for monitoring celery and see the results in the django admin page):: + + python manage.py celerycam + +* celerybeat (for running periodic tasks):: + + python manage.py celerybeat -l INFO + + +TODO +==== + +Provide a supervisord configuration diff --git a/euscanwww/djeuscan/admin.py b/euscanwww/djeuscan/admin.py index 45bdbeb..cec9244 100644 --- a/euscanwww/djeuscan/admin.py +++ b/euscanwww/djeuscan/admin.py @@ -1,22 +1,54 @@ from djeuscan.models import Package, Version, VersionLog, EuscanResult, \ - Log, WorldLog, CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer + Log, WorldLog, CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer, \ + RefreshPackageQuery, HerdAssociation, CategoryAssociation, \ + MaintainerAssociation, PackageAssociation from django.contrib import admin -admin.site.register(Herd) -admin.site.register(Maintainer) + +class EuscanResultAdmin(admin.ModelAdmin): + search_fields = ('package__name', 'package__category') + list_filter = ('datetime', ) + ordering = ["-datetime"] + + +class HerdAdmin(admin.ModelAdmin): + search_fields = ('herd', 'email') + ordering = ["herd"] + + +class MaintainerAdmin(admin.ModelAdmin): + search_fields = ('name', 'email') + ordering = ["name"] class PackageAdmin(admin.ModelAdmin): search_fields = ('category', 'name') + list_filter = ('category', ) + + +class VersionAdmin(admin.ModelAdmin): + search_fields = ('package__name', 'package__category') + list_filter = ('overlay', 'packaged', 'alive') + admin.site.register(Package, PackageAdmin) -admin.site.register(Version) +admin.site.register(Herd, HerdAdmin) +admin.site.register(Maintainer, MaintainerAdmin) + +admin.site.register(Version, VersionAdmin) admin.site.register(VersionLog) -admin.site.register(EuscanResult) + +admin.site.register(EuscanResult, EuscanResultAdmin) admin.site.register(Log) admin.site.register(WorldLog) admin.site.register(CategoryLog) admin.site.register(HerdLog) admin.site.register(MaintainerLog) + +admin.site.register(RefreshPackageQuery) +admin.site.register(HerdAssociation) +admin.site.register(CategoryAssociation) +admin.site.register(MaintainerAssociation) +admin.site.register(PackageAssociation) diff --git a/euscanwww/djeuscan/management/commands/regen_rrds.py b/euscanwww/djeuscan/management/commands/regen_rrds.py index 1c041fa..a53f44d 100644 --- a/euscanwww/djeuscan/management/commands/regen_rrds.py +++ b/euscanwww/djeuscan/management/commands/regen_rrds.py @@ -1,25 +1,10 @@ +import logging from django.core.management.base import BaseCommand -from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog -from djeuscan import charts +from djeuscan.processing import set_verbosity_level +from djeuscan.processing.regen_rrds import regen_rrds -def regen_rrds(): - """ - Regenerates the rrd database - """ - for wlog in WorldLog.objects.all(): - charts.rrd_update('world', wlog.datetime, wlog) - - for clog in CategoryLog.objects.all(): - charts.rrd_update('category-%s' % clog.category, - clog.datetime, clog) - - for hlog in HerdLog.objects.all(): - charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog) - - for mlog in MaintainerLog.objects.all(): - charts.rrd_update('maintainer-%d' % mlog.maintainer.id, - mlog.datetime, mlog) +logger = logging.getLogger(__name__) class Command(BaseCommand): @@ -27,4 +12,5 @@ class Command(BaseCommand): help = 'Regenerate rrd database' def handle(self, *args, **options): - regen_rrds() + set_verbosity_level(logger, options.get("verbosity", 1)) + regen_rrds(logger=logger) diff --git a/euscanwww/djeuscan/management/commands/scan_metadata.py b/euscanwww/djeuscan/management/commands/scan_metadata.py index d9ec08c..5decb70 100644 --- a/euscanwww/djeuscan/management/commands/scan_metadata.py +++ b/euscanwww/djeuscan/management/commands/scan_metadata.py @@ -1,142 +1,13 @@ import sys - +import logging from optparse import make_option -from gentoolkit.query import Query -from gentoolkit.errors import GentoolkitFatalError - -from django.db.transaction import commit_on_success from django.core.management.base import BaseCommand -from django.core.management.color import color_style -from django.core.exceptions import ValidationError -from djeuscan.models import Package, Herd, Maintainer +from djeuscan.processing import set_verbosity_level +from djeuscan.processing.scan_metadata import scan_metadata - -class ScanMetadata(object): - def __init__(self, quiet): - self.quiet = quiet - self.style = color_style() - - @commit_on_success - def run(self, query=None, obj=None): - matches = Query(query).find( - include_masked=True, - in_installed=False, - ) - - if not matches: - sys.stderr.write( - self.style.ERROR("Unknown package '%s'\n" % query) - ) - return - - matches = sorted(matches) - pkg = matches.pop() - if '9999' in pkg.version and len(matches): - pkg = matches.pop() - - if not obj: - obj, created = Package.objects.get_or_create( - category=pkg.category, name=pkg.name - ) - else: - created = False - - try: - obj.homepage = pkg.environment("HOMEPAGE") - obj.description = pkg.environment("DESCRIPTION") - except GentoolkitFatalError, err: - sys.stderr.write( - self.style.ERROR( - "Gentoolkit fatal error: '%s'\n" % str(err) - ) - ) - - if created and not self.quiet: - sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name)) - - if pkg.metadata: - herds = dict( - [(herd[0], herd) for herd in pkg.metadata.herds(True)] - ) - maintainers = dict( - [(m.email, m) for m in pkg.metadata.maintainers()] - ) - - existing_herds = [h.herd for h in obj.herds.all()] - new_herds = set(herds.keys()).difference(existing_herds) - old_herds = set(existing_herds).difference(herds.keys()) - - existing_maintainers = [m.email for m in obj.maintainers.all()] - new_maintainers = set( - maintainers.keys()).difference(existing_maintainers - ) - old_maintainers = set( - existing_maintainers).difference(maintainers.keys() - ) - - for herd in obj.herds.all(): - if herd.herd in old_herds: - obj.herds.remove(herd) - - for herd in new_herds: - herd = self.store_herd(*herds[herd]) - obj.herds.add(herd) - - for maintainer in obj.maintainers.all(): - if maintainer.email in old_maintainers: - obj.maintainers.remove(maintainer) - - for maintainer in new_maintainers: - maintainer = maintainers[maintainer] - try: - maintainer = self.store_maintainer( - maintainer.name, maintainer.email - ) - obj.maintainers.add(maintainer) - except ValidationError: - sys.stderr.write( - self.style.ERROR("Bad maintainer: '%s' '%s'\n" % (maintainer.name, maintainer.email)) - ) - - obj.save() - - def store_herd(self, name, email): - if not name: - name = '{nil}' - name = name.strip("\r").strip("\n").strip("\t").strip() - - herd, created = Herd.objects.get_or_create( - herd=name, - defaults={"email": email} - ) - - if created and not self.quiet: - sys.stdout.write('+ [h] %s <%s>\n' % (name, email)) - - herd.email = email - herd.save() - - return herd - - def store_maintainer(self, name, email): - if not name: - name = email - if not name: - name = '{nil}' - - maintainer, created = Maintainer.objects.get_or_create( - email=email, - defaults={"name": name} - ) - - if created: - if not self.quiet: - sys.stdout.write( - '+ [m] %s <%s>\n' % (name.encode('utf-8'), email) - ) - return maintainer +logger = logging.getLogger(__name__) class Command(BaseCommand): @@ -148,26 +19,19 @@ class Command(BaseCommand): dest='all', default=False, help='Scan all packages'), - make_option('--quiet', - action='store_true', - dest='quiet', - default=False, - help='Be quiet'), ) args = '' help = 'Scans metadata and fills database' def handle(self, *args, **options): - self.options = options - - scan_metadata = ScanMetadata(quiet=options["quiet"]) + set_verbosity_level(logger, options.get("verbosity", 1)) if options['all']: - for pkg in Package.objects.all(): - scan_metadata.run('%s/%s' % (pkg.category, pkg.name), pkg) - elif len(args) > 0: - for package in args: - scan_metadata.run(package) + packages = None + + elif len(args): + packages = [pkg for pkg in args] else: - for package in sys.stdin.readlines(): - scan_metadata.run(package[:-1]) + packages = [pkg[:-1] for pkg in sys.stdin.readlines()] + + scan_metadata(packages=packages, logger=logger) diff --git a/euscanwww/djeuscan/management/commands/scan_portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py index d5837ec..c0dd6fe 100644 --- a/euscanwww/djeuscan/management/commands/scan_portage.py +++ b/euscanwww/djeuscan/management/commands/scan_portage.py @@ -1,274 +1,13 @@ -import subprocess -import portage import sys -import os -import re - +import logging from optparse import make_option -from django.db.transaction import commit_on_success from django.core.management.base import BaseCommand -from django.core.management.color import color_style -from djeuscan.models import Package, Version, VersionLog +from djeuscan.processing import set_verbosity_level +from djeuscan.processing.scan_portage import scan_portage - -class ScanPortage(object): - def __init__(self, stdout=None, **options): - if stdout is None: - self.stdout = sys.stdout - else: - self.stdout = stdout - - self.options = options - self.style = color_style() - self._cache = {'packages': {}, 'versions': {}} - self._overlays = None - - def cache_hash_package(self, category, name): - return '%s/%s' % (category, name) - - def cache_store_package(self, package): - key = self.cache_hash_package(package.category, package.name) - self._cache['packages'][key] = package - - def cache_get_package(self, category, name): - return self._cache['packages'].get( - self.cache_hash_package(category, name) - ) - - def cache_hash_version(self, category, name, version, revision, slot, - overlay): - key = '%s/%s-%s-r%s %s %s' % (category, name, - version, revision, - slot, overlay) - return key - - def cache_get_version(self, category, name, version, revision, slot, - overlay): - key = self.cache_hash_version(category, name, version, revision, slot, - overlay) - return self._cache['versions'].get(key) - - def cache_store_version(self, version): - key = self.cache_hash_version( - version.package.category, version.package.name, version.version, - version.revision, version.slot, version.overlay - ) - self._cache['versions'][key] = version - - def overlays(self): - if self._overlays: - return self._overlays - - env = os.environ - env['OVERLAYS_LIST'] = 'all' - env['PRINT_COUNT_ALWAYS'] = 'never' - - cmd = ['eix', '-!'] - - output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\ - communicate()[0] - output = output.strip().strip('\n').split('\n') - - overlay_re = re.compile(r'^\[(?P\d+)] "(?P.*?)"') - - self._overlays = {} - - for line in output: - match = overlay_re.match(line) - if not match: - continue - self._overlays[match.group('key')] = match.group('name') - - return self._overlays - - @commit_on_success - def run(self, query=None): - env = os.environ - env['MY'] = "/-: []\n" - - cmd = ['eix', '--format', '', '--pure-packages', - '-x'] - if query: - cmd.extend(['--exact', query]) - - if self.options['all']: - if not self.options['quiet']: - self.stdout.write('Killing existing versions...') - self.stdout.flush() - Version.objects.filter(packaged=True).update(alive=False) - if not self.options['quiet']: - self.stdout.write('done\n') - - output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\ - communicate()[0] - output = output.strip().strip('\n') - - if len(output) == 0: - if not query: - return - if self.options['purge-packages']: - if not self.options['quiet']: - sys.stdout.write('- [p] %s\n' % (query)) - if '/' in query: - cat, pkg = portage.catsplit(query) - Package.objects.filter(category=cat, name=pkg).delete() - else: - Package.objects.filter(name=query).delete() - else: - sys.stderr.write( - self.style.ERROR( - "Unknown package '%s'\n" % query - ) - ) - return - - output = output.split('\n') - packages = {} - - line_re = re.compile( - r'^(?P.*?):(?P.*?) \[(?P.*?)\]$' - ) - - package = None - - for line in output: - match = line_re.match(line) - - if not match: - continue - - cpv = match.group('cpv') - slot = match.group('slot') - overlay = match.group('overlay') - - cat, pkg, ver, rev = portage.catpkgsplit(cpv) - - packages['%s/%s' % (cat, pkg)] = True - - if not package or not \ - (cat == package.category and pkg == package.name): - package = self.store_package(cat, pkg) - - self.store_version(package, cpv, slot, overlay) - - if self.options['purge-packages'] and not query: - for package in Package.objects.all(): - cp = "%s/%s" % (package.category, package.name) - if cp not in packages: - if not self.options['quiet']: - sys.stdout.write('- [p] %s\n' % (package)) - package.delete() - - def store_package(self, cat, pkg): - created = False - obj = self.cache_get_package(cat, pkg) - - if not obj: - obj, created = Package.objects.get_or_create( - category=cat, - name=pkg - ) - self.cache_store_package(obj) - - if created: - if not self.options['quiet']: - sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg)) - - # Set all versions dead, then set found versions alive and - # delete old versions - if not self.options['all']: - Version.objects.filter( - package=obj, - packaged=True - ).update(alive=False) - - return obj - - def store_version(self, package, cpv, slot, overlay): - cat, pkg, ver, rev = portage.catpkgsplit(cpv) - - overlays = self.overlays() - - if overlay in overlays: - overlay = overlays[overlay] - else: - overlay = 'gentoo' - - created = False - obj = self.cache_get_version( - package.category, package.name, ver, rev, slot, overlay - ) - if not obj: - obj, created = Version.objects.get_or_create( - package=package, slot=slot, - revision=rev, version=ver, - overlay=overlay, - defaults={"alive": True, "packaged": True} - ) - if not created: # Created objects have defaults values - obj.alive = True - obj.packaged = True - obj.save() - - if created: - self.cache_store_version(obj) - - # nothing to do (note: it can't be an upstream version because - # overlay can't be empty here) - if not created: - return - - if not self.options['quiet']: - sys.stdout.write('+ [v] %s \n' % (obj)) - - if overlay == 'gentoo': - package.n_packaged += 1 - else: - package.n_overlay += 1 - package.n_versions += 1 - package.save() - - if self.options['no-log']: - return - - VersionLog.objects.create( - package=obj.package, - action=VersionLog.VERSION_ADDED, - slot=obj.slot, - revision=obj.revision, - version=obj.version, - overlay=obj.overlay - ) - -@commit_on_success -def purge_versions(options): - # For each dead versions - for version in Version.objects.filter(packaged=True, alive=False): - if version.overlay == 'gentoo': - version.package.n_packaged -= 1 - else: - version.package.n_overlay -= 1 - version.package.n_versions -= 1 - version.package.save() - - if not options['quiet']: - sys.stdout.write('- [v] %s\n' % (version)) - - if options['no-log']: - continue - - VersionLog.objects.create( - package=version.package, - action=VersionLog.VERSION_REMOVED, - slot=version.slot, - revision=version.revision, - version=version.version, - overlay=version.overlay - ) - - Version.objects.filter(packaged=True, alive=False).delete() +logger = logging.getLogger(__name__) class Command(BaseCommand): @@ -301,43 +40,26 @@ class Command(BaseCommand): default=False, help=('Prefetch all versions and packages from DB to ' 'speedup full scan process.')), - make_option('--quiet', - action='store_true', - dest='quiet', - default=False, - help='Be quiet'), ) args = '[package package ...]' help = 'Scans portage tree and fills database' def handle(self, *args, **options): - scan_portage = ScanPortage(stdout=self.stdout, **options) - - if not options['quiet']: - self.stdout.write('Scanning portage tree...\n') - - if options['prefetch']: - if not options['quiet']: - self.stdout.write('Prefetching objects...') - self.stdout.flush() - for package in Package.objects.all(): - scan_portage.cache_store_package(package) - for version in Version.objects.select_related('package').all(): - scan_portage.cache_store_version(version) - if not options['quiet']: - self.stdout.write('done\n') + set_verbosity_level(logger, options.get("verbosity", 1)) if options['all']: - scan_portage.run() + packages = None + elif len(args): - for package in args: - scan_portage.run(package) + packages = [pkg for pkg in args] else: - for package in sys.stdin.readlines(): - scan_portage.run(package[:-1]) + packages = [pkg[:-1] for pkg in sys.stdin.readlines()] - if options['purge-versions']: - purge_versions(options) - - if not options['quiet']: - self.stdout.write('Done.\n') + scan_portage( + packages=packages, + no_log=options["no-log"], + purge_packages=options["purge-packages"], + purge_versions=options["purge-versions"], + prefetch=options["prefetch"], + logger=logger, + ) diff --git a/euscanwww/djeuscan/management/commands/scan_upstream.py b/euscanwww/djeuscan/management/commands/scan_upstream.py index a72f0b6..f185c38 100644 --- a/euscanwww/djeuscan/management/commands/scan_upstream.py +++ b/euscanwww/djeuscan/management/commands/scan_upstream.py @@ -1,151 +1,13 @@ -import subprocess -import portage +import logging import sys -import re -from StringIO import StringIO from optparse import make_option -from collections import defaultdict -from django.utils import timezone -from django.db.transaction import commit_on_success from django.core.management.base import BaseCommand -from djeuscan.models import Package, Version, EuscanResult, VersionLog +from djeuscan.processing import set_verbosity_level +from djeuscan.processing.scan_upstream import scan_upstream - -class ScanUpstream(object): - def __init__(self, options=None): - if options is None: - self.options = defaultdict(None) - else: - self.options = options - - def run(self, packages=None): - for package in packages: - cmd = ['euscan', package] - - fp = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output = StringIO(fp.communicate()[0]) - - self.parse_output(output) - - def store_result(self, package, log): - # Remove previous logs - EuscanResult.objects.filter(package=package).delete() - - obj = EuscanResult() - obj.package = package - obj.result = log - obj.datetime = timezone.now() - obj.save() - - def store_package(self, cpv): - cat, pkg, ver, rev = portage.catpkgsplit(cpv) - - obj, created = Package.objects.get_or_create(category=cat, name=pkg) - - if created and not self.options['quiet']: - sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg)) - - # Set all versions dead, then set found versions alive and - # delete old versions - Version.objects.filter(package=obj, packaged=False).update(alive=False) - - return obj - - def store_version(self, package, ver, url): - obj, created = Version.objects.get_or_create( - package=package, slot='', revision='r0', version=ver, overlay='', - defaults={"alive": True, "urls": url, "packaged": False} - ) - if not created: - obj.alive = True - obj.urls = url - obj.packaged = False - obj.save() - - # If it's not a new version, just update the object and continue - if not created: - return - - if not self.options['quiet']: - sys.stdout.write('+ [u] %s %s\n' % (obj, url)) - - VersionLog.objects.create( - package=package, - action=VersionLog.VERSION_ADDED, - slot='', - revision='r0', - version=ver, - overlay='' - ) - - package.n_versions += 1 - package.save() - - @commit_on_success - def parse_output(self, output): - from portage.versions import _cp - - if type(_cp) == dict: - _cp = _cp["dots_allowed_in_PN"] - - package_re = re.compile( - r'^ \* (?P' + _cp + ') \[(?P.*?)\]$' - ) - version_re = re.compile( - r'^Upstream Version: (?P.*?) (?P.*?)$' - ) - - package = None - log = "" - - while True: - line = output.readline() - if line == '': - break - match = package_re.match(line) - if match: - if package: - self.store_result(package, log) - - cpv = match.group('cpv') - package = self.store_package(cpv) - log = line - continue - - log += line - - match = version_re.match(line) - if match: - ver = match.group('ver') - url = match.group('url') - self.store_version(package, ver, url) - - if package: - self.store_result(package, log) - - -@commit_on_success -def purge_versions(options): - # For each dead versions - for version in Version.objects.filter(packaged=False, alive=False): - VersionLog.objects.create( - package=version.package, - action=VersionLog.VERSION_REMOVED, - slot=version.slot, - revision=version.revision, - version=version.version, - overlay=version.overlay - ) - - version.package.n_versions -= 1 - version.package.save() - - if not options['quiet']: - sys.stdout.write('- [u] %s %s\n' % (version, version.urls)) - Version.objects.filter(packaged=False, alive=False).delete() +logger = logging.getLogger(__name__) class Command(BaseCommand): @@ -162,41 +24,23 @@ class Command(BaseCommand): dest='purge-versions', default=False, help='Purge old versions'), - make_option('--quiet', - action='store_true', - dest='quiet', - default=False, - help='Be quiet'), ) args = '' help = 'Scans metadata and fills database' def handle(self, *args, **options): - scan_upstream = ScanUpstream(options) - - if not args and not options['all']: - scan_upstream.parse_output(sys.stdin) - if options['purge-versions']: - purge_versions(options) - return - - if not options['quiet']: - self.stdout.write('Scanning upstream...\n') - - packages = [] + set_verbosity_level(logger, options.get("verbosity", 1)) if options['all']: - for pkg in Package.objects.all(): - packages.append('%s/%s' % (pkg.category, pkg.name)) - elif args: - packages = list(args) + packages = None + + elif len(args): + packages = [pkg for pkg in args] else: - packages = [package[:-1] for package in sys.stdin.readlines()] + packages = [pkg[:-1] for pkg in sys.stdin.readlines()] - scan_upstream.run(packages) - - if options['purge-versions']: - purge_versions(options) - - if not options['quiet']: - self.stdout.write('Done.\n') + scan_upstream( + packages=packages, + purge_versions=options["purge-versions"], + logger=logger, + ) diff --git a/euscanwww/djeuscan/management/commands/update_counters.py b/euscanwww/djeuscan/management/commands/update_counters.py index 0b67a39..100d5a7 100644 --- a/euscanwww/djeuscan/management/commands/update_counters.py +++ b/euscanwww/djeuscan/management/commands/update_counters.py @@ -1,185 +1,12 @@ -import sys +import logging from optparse import make_option -from django.db.transaction import commit_on_success from django.core.management.base import BaseCommand -from django.utils import timezone -from djeuscan.models import Package, Herd, Maintainer, Version -from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog -from djeuscan import charts +from djeuscan.processing import set_verbosity_level +from djeuscan.processing.update_counters import update_counters -from distutils.version import StrictVersion, LooseVersion - - -def compare_versions(version1, version2): - try: - return cmp(StrictVersion(version1), StrictVersion(version2)) - # in case of abnormal version number, fall back to LooseVersion - except ValueError: - return cmp(LooseVersion(version1), LooseVersion(version2)) - - -def add_safe(storage, key): - if key not in storage: - storage[key] = 1 - else: - storage[key] += 1 - - -def add_last_ver(storage, version): - key = version['package_id'] - if key not in storage: - storage[key] = version - return - if version['version'].startswith('9999'): - return - if compare_versions(storage[key]['version'], - version['version']) < 0: - storage[key] = version - - -@commit_on_success -def update_counters(stdout=None, **options): - if stdout is None: - stdout = sys.stdout - - now = timezone.now() - - categories = {} - herds = {} - maintainers = {} - - wlog = None - - if not options['nolog']: - wlog = WorldLog() - wlog.datetime = now - - for cat in Package.objects.values('category').distinct(): - clog = CategoryLog() - clog.datetime = now - clog.category = cat['category'] - categories[clog.category] = clog - - for herd in Herd.objects.all(): - hlog = HerdLog() - hlog.datetime = now - hlog.herd = herd - herds[herd.id] = hlog - - for maintainer in Maintainer.objects.all(): - mlog = MaintainerLog() - mlog.datetime = now - mlog.maintainer = maintainer - maintainers[maintainer.id] = mlog - - package_queryset = Package.objects.all() - - n_versions = {} - n_packaged = {} - n_overlay = {} - - last_versions_gentoo = {} - last_versions_overlay = {} - last_versions_upstream = {} - - if not options['fast']: - attrs = ['id', 'version', 'overlay', 'packaged', 'package_id'] - for version in Version.objects.all().values(*attrs): - overlay, packaged = version['overlay'], version['packaged'] - package_id = version['package_id'] - - add_safe(n_versions, package_id) - - if not packaged: - add_last_ver(last_versions_upstream, version) - continue - if overlay == 'gentoo': - add_safe(n_packaged, package_id) - add_last_ver(last_versions_gentoo, version) - else: - add_safe(n_overlay, package_id) - add_last_ver(last_versions_overlay, version) - - for package in package_queryset.select_related('herds', 'maintainers'): - if not options['fast']: - package.n_versions = n_versions.get(package.id, 0) - package.n_packaged = n_packaged.get(package.id, 0) - package.n_overlay = n_overlay.get(package.id, 0) - - default = {'id': None} - package.last_version_gentoo_id = last_versions_gentoo.get( - package.id, default - )['id'] - package.last_version_overlay_id = last_versions_overlay.get( - package.id, default - )['id'] - package.last_version_upstream_id = last_versions_upstream.get( - package.id, default - )['id'] - - package.save() - - n_packages_gentoo = int(package.n_packaged == package.n_versions) - n_packages_overlay = int(package.n_overlay and package.n_packaged \ - + package.n_overlay == package.n_versions) - n_packages_outdated = int(package.n_packaged + package.n_overlay \ - < package.n_versions) - - def update_row(storage, key): - storage[key].n_packages_gentoo += n_packages_gentoo - storage[key].n_packages_overlay += n_packages_overlay - storage[key].n_packages_outdated += n_packages_outdated - - storage[key].n_versions_gentoo += package.n_packaged - storage[key].n_versions_overlay += package.n_overlay - storage[key].n_versions_upstream += package.n_versions - \ - package.n_packaged - \ - package.n_overlay - - def update_log(storage, qs): - for row in qs: - update_row(storage, row['id']) - - if not options['nolog']: - update_log(herds, package.herds.all().values('id')) - update_log(maintainers, package.maintainers.all().values('id')) - update_row(categories, package.category) - - wlog.n_packages_gentoo += n_packages_gentoo - wlog.n_packages_overlay += n_packages_overlay - wlog.n_packages_outdated += n_packages_outdated - - wlog.n_versions_gentoo += package.n_packaged - wlog.n_versions_overlay += package.n_overlay - wlog.n_versions_upstream += package.n_versions - \ - package.n_packaged - \ - package.n_overlay - - if options['nolog']: - return - - for clog in categories.values(): - if not options['quiet']: - stdout.write('+ [cl] %s\n' % clog) - charts.rrd_update('category-%s' % clog.category, now, clog) - clog.save() - - for hlog in herds.values(): - if not options['quiet']: - stdout.write('+ [hl] %s\n' % hlog) - charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog) - hlog.save() - - for mlog in maintainers.values(): - if not options['quiet']: - stdout.write('+ [ml] %s\n' % mlog) - charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog) - mlog.save() - - charts.rrd_update('world', now, wlog) - wlog.save() +logger = logging.getLogger(__name__) class Command(BaseCommand): @@ -187,11 +14,6 @@ class Command(BaseCommand): help = 'Update counters' option_list = BaseCommand.option_list + ( - make_option('--quiet', - action='store_true', - dest='quiet', - default=False, - help='Be quiet'), make_option('--fast', action='store_true', dest='fast', @@ -205,4 +27,9 @@ class Command(BaseCommand): ) def handle(self, *args, **options): - update_counters(stdout=self.stdout, **options) + set_verbosity_level(logger, options.get("verbosity", 1)) + update_counters( + fast=options["fast"], + nolog=options["nolog"], + logger=logger, + ) diff --git a/euscanwww/djeuscan/managers.py b/euscanwww/djeuscan/managers.py index 2172602..0d4c544 100644 --- a/euscanwww/djeuscan/managers.py +++ b/euscanwww/djeuscan/managers.py @@ -35,7 +35,6 @@ ANNOTATE_DICT = {name: models.Sum(name) class PackageMixin(object): - for_maintainer = _gen_for_function('maintainers') for_herd = _gen_for_function('herds') for_category = _gen_for_function('category') diff --git a/euscanwww/djeuscan/migrations/0003_auto__add_refreshpackagequery__chg_field_package_last_version_gentoo__.py b/euscanwww/djeuscan/migrations/0003_auto__add_refreshpackagequery__chg_field_package_last_version_gentoo__.py new file mode 100644 index 0000000..2724c09 --- /dev/null +++ b/euscanwww/djeuscan/migrations/0003_auto__add_refreshpackagequery__chg_field_package_last_version_gentoo__.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'RefreshPackageQuery' + db.create_table('djeuscan_refreshpackagequery', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('query', self.gf('django.db.models.fields.CharField')(unique=True, max_length=256)), + ('priority', self.gf('django.db.models.fields.IntegerField')(default=0)), + )) + db.send_create_signal('djeuscan', ['RefreshPackageQuery']) + + + # Changing field 'Package.last_version_gentoo' + db.alter_column('djeuscan_package', 'last_version_gentoo_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['djeuscan.Version'])) + + # Changing field 'Package.last_version_overlay' + db.alter_column('djeuscan_package', 'last_version_overlay_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['djeuscan.Version'])) + + # Changing field 'Package.last_version_upstream' + db.alter_column('djeuscan_package', 'last_version_upstream_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['djeuscan.Version'])) + def backwards(self, orm): + # Deleting model 'RefreshPackageQuery' + db.delete_table('djeuscan_refreshpackagequery') + + + # Changing field 'Package.last_version_gentoo' + db.alter_column('djeuscan_package', 'last_version_gentoo_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['djeuscan.Version'])) + + # Changing field 'Package.last_version_overlay' + db.alter_column('djeuscan_package', 'last_version_overlay_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['djeuscan.Version'])) + + # Changing field 'Package.last_version_upstream' + db.alter_column('djeuscan_package', 'last_version_upstream_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['djeuscan.Version'])) + models = { + 'djeuscan.categorylog': { + 'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.euscanresult': { + 'Meta': {'object_name': 'EuscanResult'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'djeuscan.herd': { + 'Meta': {'object_name': 'Herd'}, + 'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'djeuscan.herdlog': { + 'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.log': { + 'Meta': {'object_name': 'Log'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'djeuscan.maintainer': { + 'Meta': {'object_name': 'Maintainer'}, + 'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.maintainerlog': { + 'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}) + }, + 'djeuscan.package': { + 'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}), + 'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}), + 'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.refreshpackagequery': { + 'Meta': {'object_name': 'RefreshPackageQuery'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'}) + }, + 'djeuscan.version': { + 'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'}, + 'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.versionlog': { + 'Meta': {'object_name': 'VersionLog'}, + 'action': ('django.db.models.fields.IntegerField', [], {}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.worldlog': { + 'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + } + } + + complete_apps = ['djeuscan'] \ No newline at end of file diff --git a/euscanwww/djeuscan/migrations/0004_auto__add_categoryassociation__add_unique_categoryassociation_user_cat.py b/euscanwww/djeuscan/migrations/0004_auto__add_categoryassociation__add_unique_categoryassociation_user_cat.py new file mode 100644 index 0000000..888fefa --- /dev/null +++ b/euscanwww/djeuscan/migrations/0004_auto__add_categoryassociation__add_unique_categoryassociation_user_cat.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'CategoryAssociation' + db.create_table('djeuscan_categoryassociation', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + ('category', self.gf('django.db.models.fields.CharField')(max_length=128)), + )) + db.send_create_signal('djeuscan', ['CategoryAssociation']) + + # Adding unique constraint on 'CategoryAssociation', fields ['user', 'category'] + db.create_unique('djeuscan_categoryassociation', ['user_id', 'category']) + + # Adding model 'PackageAssociation' + db.create_table('djeuscan_packageassociation', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + ('package', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Package'])), + )) + db.send_create_signal('djeuscan', ['PackageAssociation']) + + # Adding unique constraint on 'PackageAssociation', fields ['user', 'package'] + db.create_unique('djeuscan_packageassociation', ['user_id', 'package_id']) + + # Adding model 'HerdAssociation' + db.create_table('djeuscan_herdassociation', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + ('herd', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Herd'])), + )) + db.send_create_signal('djeuscan', ['HerdAssociation']) + + # Adding unique constraint on 'HerdAssociation', fields ['user', 'herd'] + db.create_unique('djeuscan_herdassociation', ['user_id', 'herd_id']) + + # Adding model 'MaintainerAssociation' + db.create_table('djeuscan_maintainerassociation', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + ('maintainer', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Maintainer'])), + )) + db.send_create_signal('djeuscan', ['MaintainerAssociation']) + + # Adding unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer'] + db.create_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id']) + + def backwards(self, orm): + # Removing unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer'] + db.delete_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id']) + + # Removing unique constraint on 'HerdAssociation', fields ['user', 'herd'] + db.delete_unique('djeuscan_herdassociation', ['user_id', 'herd_id']) + + # Removing unique constraint on 'PackageAssociation', fields ['user', 'package'] + db.delete_unique('djeuscan_packageassociation', ['user_id', 'package_id']) + + # Removing unique constraint on 'CategoryAssociation', fields ['user', 'category'] + db.delete_unique('djeuscan_categoryassociation', ['user_id', 'category']) + + # Deleting model 'CategoryAssociation' + db.delete_table('djeuscan_categoryassociation') + + # Deleting model 'PackageAssociation' + db.delete_table('djeuscan_packageassociation') + + # Deleting model 'HerdAssociation' + db.delete_table('djeuscan_herdassociation') + + # Deleting model 'MaintainerAssociation' + db.delete_table('djeuscan_maintainerassociation') + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'djeuscan.categoryassociation': { + 'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.categorylog': { + 'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.euscanresult': { + 'Meta': {'object_name': 'EuscanResult'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'djeuscan.herd': { + 'Meta': {'object_name': 'Herd'}, + 'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'djeuscan.herdassociation': { + 'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.herdlog': { + 'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.log': { + 'Meta': {'object_name': 'Log'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'djeuscan.maintainer': { + 'Meta': {'object_name': 'Maintainer'}, + 'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.maintainerassociation': { + 'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.maintainerlog': { + 'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}) + }, + 'djeuscan.package': { + 'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}), + 'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}), + 'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.packageassociation': { + 'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.refreshpackagequery': { + 'Meta': {'object_name': 'RefreshPackageQuery'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'}) + }, + 'djeuscan.version': { + 'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'}, + 'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.versionlog': { + 'Meta': {'object_name': 'VersionLog'}, + 'action': ('django.db.models.fields.IntegerField', [], {}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.worldlog': { + 'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + } + } + + complete_apps = ['djeuscan'] \ No newline at end of file diff --git a/euscanwww/djeuscan/migrations/0005_auto__add_field_version_version_type__add_field_version_handler__add_f.py b/euscanwww/djeuscan/migrations/0005_auto__add_field_version_version_type__add_field_version_handler__add_f.py new file mode 100644 index 0000000..64eda3c --- /dev/null +++ b/euscanwww/djeuscan/migrations/0005_auto__add_field_version_version_type__add_field_version_handler__add_f.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'Version.version_type' + db.add_column('djeuscan_version', 'version_type', + self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True), + keep_default=False) + + # Adding field 'Version.handler' + db.add_column('djeuscan_version', 'handler', + self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True), + keep_default=False) + + # Adding field 'Version.confidence' + db.add_column('djeuscan_version', 'confidence', + self.gf('django.db.models.fields.IntegerField')(default=0), + keep_default=False) + + def backwards(self, orm): + # Deleting field 'Version.version_type' + db.delete_column('djeuscan_version', 'version_type') + + # Deleting field 'Version.handler' + db.delete_column('djeuscan_version', 'handler') + + # Deleting field 'Version.confidence' + db.delete_column('djeuscan_version', 'confidence') + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'djeuscan.categoryassociation': { + 'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.categorylog': { + 'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.euscanresult': { + 'Meta': {'object_name': 'EuscanResult'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'djeuscan.herd': { + 'Meta': {'object_name': 'Herd'}, + 'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'djeuscan.herdassociation': { + 'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.herdlog': { + 'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.log': { + 'Meta': {'object_name': 'Log'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'djeuscan.maintainer': { + 'Meta': {'object_name': 'Maintainer'}, + 'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.maintainerassociation': { + 'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.maintainerlog': { + 'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}) + }, + 'djeuscan.package': { + 'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}), + 'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}), + 'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.packageassociation': { + 'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.refreshpackagequery': { + 'Meta': {'object_name': 'RefreshPackageQuery'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'}) + }, + 'djeuscan.version': { + 'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'}, + 'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'version_type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.versionlog': { + 'Meta': {'object_name': 'VersionLog'}, + 'action': ('django.db.models.fields.IntegerField', [], {}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.worldlog': { + 'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + } + } + + complete_apps = ['djeuscan'] \ No newline at end of file diff --git a/euscanwww/djeuscan/migrations/0006_auto__add_field_euscanresult_scan_time__add_field_euscanresult_ebuild.py b/euscanwww/djeuscan/migrations/0006_auto__add_field_euscanresult_scan_time__add_field_euscanresult_ebuild.py new file mode 100644 index 0000000..9e4bcce --- /dev/null +++ b/euscanwww/djeuscan/migrations/0006_auto__add_field_euscanresult_scan_time__add_field_euscanresult_ebuild.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'EuscanResult.scan_time' + db.add_column('djeuscan_euscanresult', 'scan_time', + self.gf('django.db.models.fields.FloatField')(null=True, blank=True), + keep_default=False) + + # Adding field 'EuscanResult.ebuild' + db.add_column('djeuscan_euscanresult', 'ebuild', + self.gf('django.db.models.fields.CharField')(default='', max_length=256, blank=True), + keep_default=False) + + def backwards(self, orm): + # Deleting field 'EuscanResult.scan_time' + db.delete_column('djeuscan_euscanresult', 'scan_time') + + # Deleting field 'EuscanResult.ebuild' + db.delete_column('djeuscan_euscanresult', 'ebuild') + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'djeuscan.categoryassociation': { + 'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.categorylog': { + 'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.euscanresult': { + 'Meta': {'object_name': 'EuscanResult'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}) + }, + 'djeuscan.herd': { + 'Meta': {'object_name': 'Herd'}, + 'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'djeuscan.herdassociation': { + 'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.herdlog': { + 'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.log': { + 'Meta': {'object_name': 'Log'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'djeuscan.maintainer': { + 'Meta': {'object_name': 'Maintainer'}, + 'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.maintainerassociation': { + 'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.maintainerlog': { + 'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}) + }, + 'djeuscan.package': { + 'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}), + 'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}), + 'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.packageassociation': { + 'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.refreshpackagequery': { + 'Meta': {'object_name': 'RefreshPackageQuery'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'}) + }, + 'djeuscan.version': { + 'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'}, + 'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'version_type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.versionlog': { + 'Meta': {'object_name': 'VersionLog'}, + 'action': ('django.db.models.fields.IntegerField', [], {}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.worldlog': { + 'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + } + } + + complete_apps = ['djeuscan'] \ No newline at end of file diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py index 9445af4..e5abd81 100644 --- a/euscanwww/djeuscan/models.py +++ b/euscanwww/djeuscan/models.py @@ -2,6 +2,8 @@ from django.db import models from django.core.validators import RegexValidator, validate_email, URLValidator from django.core.exceptions import ValidationError +from django.contrib.auth.models import User + from djeuscan.managers import PackageManager, VersionLogManager, \ EuscanResultManager @@ -115,16 +117,19 @@ class Version(models.Model): """ package = models.ForeignKey(Package) - slot = models.CharField(max_length=128, blank=True, default='') + slot = models.CharField(max_length=128, blank=True, default="") revision = models.CharField(max_length=128) version = models.CharField(max_length=128) packaged = models.BooleanField() - overlay = models.CharField(max_length=128, blank=True, - default='gentoo', db_index=True, - validators=[validate_name]) + overlay = models.CharField(max_length=128, default='gentoo', db_index=True, + validators=[validate_name], blank=True) urls = models.TextField(blank=True) alive = models.BooleanField(default=True, db_index=True) + version_type = models.CharField(max_length=128, blank=True) + handler = models.CharField(max_length=128, blank=True) + confidence = models.IntegerField(default=0) + class Meta: unique_together = ['package', 'slot', 'revision', 'version', 'overlay'] @@ -149,12 +154,12 @@ class VersionLog(models.Model): package = models.ForeignKey(Package) datetime = models.DateTimeField(auto_now_add=True) - slot = models.CharField(max_length=128, blank=True, default='') + slot = models.CharField(max_length=128, blank=True, default="") revision = models.CharField(max_length=128) version = models.CharField(max_length=128) packaged = models.BooleanField() - overlay = models.CharField(max_length=128, blank=True, - default='gentoo', validators=[validate_name]) + overlay = models.CharField(max_length=128, default='gentoo', + validators=[validate_name], blank=True) action = models.IntegerField(choices=VERSION_ACTIONS) objects = VersionLogManager() @@ -182,6 +187,9 @@ class EuscanResult(models.Model): datetime = models.DateTimeField() result = models.TextField(blank=True) + scan_time = models.FloatField(null=True, blank=True) + ebuild = models.CharField(blank=True, max_length=256) + objects = EuscanResultManager() class Meta: @@ -191,6 +199,11 @@ class EuscanResult(models.Model): self.full_clean() super(EuscanResult, self).save(*args, **kwargs) + def __unicode__(self): + return '[%s] %s/%s' % ( + self.datetime, self.package.category, self.package.name + ) + class Log(models.Model): """ @@ -253,3 +266,55 @@ class MaintainerLog(Log): def __unicode__(self): return u'%s %s' % (self.maintainer, Log.__unicode__(self)) + + +class RefreshPackageQuery(models.Model): + query = models.CharField(max_length=256, unique=True) + priority = models.IntegerField(default=0) + + def __unicode__(self): + return u'[%d] %s' % (self.priority, self.query) + + +class HerdAssociation(models.Model): + user = models.ForeignKey(User) + herd = models.ForeignKey(Herd) + + class Meta: + unique_together = ['user', 'herd'] + + def __unicode__(self): + return u'[%s] %s' % (self.user, self.herd) + + +class MaintainerAssociation(models.Model): + user = models.ForeignKey(User) + maintainer = models.ForeignKey(Maintainer) + + class Meta: + unique_together = ['user', 'maintainer'] + + def __unicode__(self): + return u'[%s] %s' % (self.user, self.maintainer) + + +class PackageAssociation(models.Model): + user = models.ForeignKey(User) + package = models.ForeignKey(Package) + + class Meta: + unique_together = ['user', 'package'] + + def __unicode__(self): + return u'[%s] %s' % (self.user, self.package) + + +class CategoryAssociation(models.Model): + user = models.ForeignKey(User) + category = models.CharField(max_length=128, validators=[validate_category]) + + class Meta: + unique_together = ['user', 'category'] + + def __unicode__(self): + return u'[%s] %s' % (self.user, self.category) diff --git a/euscanwww/djeuscan/processing/__init__.py b/euscanwww/djeuscan/processing/__init__.py new file mode 100644 index 0000000..4a203b7 --- /dev/null +++ b/euscanwww/djeuscan/processing/__init__.py @@ -0,0 +1,31 @@ +import logging + + +class FakeLogger(object): + def __getattr__(self, key): + return lambda *x, **y: None + + +def set_verbosity_level(logger, verbosity): + try: + verbosity = int(verbosity) + except (ValueError, TypeError): + return logger + + levels = { + 0: logging.DEBUG, + 1: logging.INFO, + 2: logging.WARNING, + 3: logging.ERROR, + 4: logging.CRITICAL + } + + if verbosity < 0: + verbosity = 0 + + if verbosity > 4: + verbosity = 4 + + logger.setLevel(levels[verbosity]) + + return logger diff --git a/euscanwww/djeuscan/processing/regen_rrds.py b/euscanwww/djeuscan/processing/regen_rrds.py new file mode 100644 index 0000000..f578aa7 --- /dev/null +++ b/euscanwww/djeuscan/processing/regen_rrds.py @@ -0,0 +1,31 @@ +from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog +from djeuscan import charts + +from djeuscan.processing import FakeLogger + + +def regen_rrds(logger=None): + """ + Regenerates the rrd database + """ + + if logger is None: + logger = FakeLogger() + + logger.info("Regenering RRDs for world") + for wlog in WorldLog.objects.all(): + charts.rrd_update('world', wlog.datetime, wlog) + + logger.info("Regenering RRDs for categories") + for clog in CategoryLog.objects.all(): + charts.rrd_update('category-%s' % clog.category, + clog.datetime, clog) + + logger.info("Regenering RRDs for herds") + for hlog in HerdLog.objects.all(): + charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog) + + logger.info("Regenering RRDs for maintainers") + for mlog in MaintainerLog.objects.all(): + charts.rrd_update('maintainer-%d' % mlog.maintainer.id, + mlog.datetime, mlog) diff --git a/euscanwww/djeuscan/processing/scan_metadata.py b/euscanwww/djeuscan/processing/scan_metadata.py new file mode 100644 index 0000000..c834327 --- /dev/null +++ b/euscanwww/djeuscan/processing/scan_metadata.py @@ -0,0 +1,148 @@ +from gentoolkit.query import Query +from gentoolkit.errors import GentoolkitFatalError + +from django.db.transaction import commit_on_success +from django.core.management.color import color_style +from django.core.exceptions import ValidationError + +from djeuscan.models import Package, Herd, Maintainer +from djeuscan.processing import FakeLogger + + +class ScanMetadata(object): + def __init__(self, logger=None): + self.style = color_style() + self.logger = logger or FakeLogger() + + @commit_on_success + def scan(self, query=None, obj=None): + matches = Query(query).find( + include_masked=True, + in_installed=False, + ) + + if not matches: + self.logger.error( + self.style.ERROR("Unknown package '%s'" % query) + ) + return + + matches = sorted(matches) + pkg = matches.pop() + if '9999' in pkg.version and len(matches): + pkg = matches.pop() + + if not obj: + obj, created = Package.objects.get_or_create( + category=pkg.category, name=pkg.name + ) + else: + created = False + + try: + obj.homepage = pkg.environment("HOMEPAGE") + obj.description = pkg.environment("DESCRIPTION") + except GentoolkitFatalError, err: + self.logger.error( + self.style.ERROR( + "Gentoolkit fatal error: '%s'" % str(err) + ) + ) + + if created: + self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name)) + + if pkg.metadata: + herds = dict( + [(herd[0], herd) for herd in pkg.metadata.herds(True)] + ) + maintainers = dict( + [(m.email, m) for m in pkg.metadata.maintainers()] + ) + + existing_herds = [h.herd for h in obj.herds.all()] + new_herds = set(herds.keys()).difference(existing_herds) + old_herds = set(existing_herds).difference(herds.keys()) + + existing_maintainers = [m.email for m in obj.maintainers.all()] + new_maintainers = set( + maintainers.keys()).difference(existing_maintainers + ) + old_maintainers = set( + existing_maintainers).difference(maintainers.keys() + ) + + for herd in obj.herds.all(): + if herd.herd in old_herds: + obj.herds.remove(herd) + + for herd in new_herds: + herd = self.store_herd(*herds[herd]) + obj.herds.add(herd) + + for maintainer in obj.maintainers.all(): + if maintainer.email in old_maintainers: + obj.maintainers.remove(maintainer) + + for maintainer in new_maintainers: + maintainer = maintainers[maintainer] + try: + maintainer = self.store_maintainer( + maintainer.name, maintainer.email + ) + obj.maintainers.add(maintainer) + except ValidationError: + self.logger.error( + self.style.ERROR("Bad maintainer: '%s' '%s'" % \ + (maintainer.name, maintainer.email)) + ) + obj.save() + return True + + def store_herd(self, name, email): + if not name: + name = '{nil}' + name = name.strip("\r").strip("\n").strip("\t").strip() + + herd, created = Herd.objects.get_or_create( + herd=name, + defaults={"email": email} + ) + + if created: + self.logger.info('+ [h] %s <%s>' % (name, email)) + + herd.email = email + herd.save() + + return herd + + def store_maintainer(self, name, email): + if not name: + name = email + if not name: + name = '{nil}' + + maintainer, created = Maintainer.objects.get_or_create( + email=email, + defaults={"name": name} + ) + + if created: + self.logger.info( + '+ [m] %s <%s>' % (name.encode('utf-8'), email) + ) + return maintainer + + +def scan_metadata(packages=None, logger=None): + scan_handler = ScanMetadata(logger=logger) + if not packages: + packages = Package.objects.all() + + for pkg in packages: + if isinstance(pkg, Package): + result = scan_handler.scan('%s/%s' % (pkg.category, pkg.name), pkg) + else: + result = scan_handler.scan(pkg) + return result diff --git a/euscanwww/djeuscan/processing/scan_portage.py b/euscanwww/djeuscan/processing/scan_portage.py new file mode 100644 index 0000000..29f5974 --- /dev/null +++ b/euscanwww/djeuscan/processing/scan_portage.py @@ -0,0 +1,307 @@ +import subprocess +import portage +import os +import re +from xml.dom.minidom import parseString + +from django.db.transaction import commit_on_success +from django.core.management.color import color_style + +from euscan.helpers import get_version_type + +from djeuscan.processing import FakeLogger +from djeuscan.models import Package, Version, VersionLog + + +class ScanPortage(object): + def __init__(self, logger=None, no_log=False, purge_packages=False, + kill_versions=False): + self.logger = logger or FakeLogger() + self.no_log = no_log + self.purge_packages = purge_packages + self.kill_versions = kill_versions + + self.style = color_style() + self._cache = {'packages': {}, 'versions': {}} + self._overlays = None + + def cache_hash_package(self, category, name): + return '%s/%s' % (category, name) + + def cache_store_package(self, package): + key = self.cache_hash_package(package.category, package.name) + self._cache['packages'][key] = package + + def cache_get_package(self, category, name): + return self._cache['packages'].get( + self.cache_hash_package(category, name) + ) + + def cache_hash_version(self, category, name, version, revision, slot, + overlay): + key = '%s/%s-%s-r%s %s %s' % (category, name, + version, revision, + slot, overlay) + return key + + def cache_get_version(self, category, name, version, revision, slot, + overlay): + key = self.cache_hash_version(category, name, version, revision, slot, + overlay) + return self._cache['versions'].get(key) + + def cache_store_version(self, version): + key = self.cache_hash_version( + version.package.category, version.package.name, version.version, + version.revision, version.slot, version.overlay + ) + self._cache['versions'][key] = version + + def overlays(self): + if self._overlays: + return self._overlays + + env = os.environ + env['OVERLAYS_LIST'] = 'all' + env['PRINT_COUNT_ALWAYS'] = 'never' + + cmd = ['eix', '-!'] + + output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\ + communicate()[0] + output = output.strip().strip('\n').split('\n') + + overlay_re = re.compile(r'^\[(?P\d+)] "(?P.*?)"') + + self._overlays = {} + + for line in output: + match = overlay_re.match(line) + if not match: + continue + self._overlays[match.group('key')] = match.group('name') + + return self._overlays + + def scan(self, query=None): + if self.purge_packages: + with commit_on_success(): + for package in Package.objects.all(): + self.logger.info('- [p] %s' % (package)) + package.delete() + + cmd = ['eix', '--xml', '--pure-packages', '-x'] + if query: + cmd.extend(['--exact', query]) + + if self.kill_versions: + self.logger.info('Killing existing versions...') + Version.objects.filter(packaged=True).update(alive=False) + self.logger.info('done') + + output = subprocess.Popen(cmd, stdout=subprocess.PIPE).\ + communicate()[0] + + if len(output) == 0: + if not query: + return + if self.purge_packages: + self.logger.info('- [p] %s' % (query)) + if '/' in query: + cat, pkg = portage.catsplit(query) + Package.objects.filter(category=cat, name=pkg).delete() + else: + Package.objects.filter(name=query).delete() + else: + self.logger.error( + self.style.ERROR( + "Unknown package '%s'" % query + ) + ) + return + + dom = parseString(output) + + for category_tag in dom.getElementsByTagName("category"): + for package_tag in category_tag.getElementsByTagName("package"): + cat = category_tag.getAttribute("name") + pkg = package_tag.getAttribute("name") + homepage_tags = package_tag.getElementsByTagName("homepage") + try: + homepage = homepage_tags[0].firstChild.nodeValue + except (IndexError, AttributeError): + homepage = "" + desc_tags = package_tag.getElementsByTagName("description") + try: + desc = desc_tags[0].firstChild.nodeValue + except (IndexError, AttributeError): + desc = "" + + with commit_on_success(): + package = self.store_package(cat, pkg, homepage, desc) + + for version_tag in package_tag.\ + getElementsByTagName("version"): + cpv = "%s/%s-%s" % (cat, pkg, + version_tag.getAttribute("id")) + slot = version_tag.getAttribute("slot") + overlay = version_tag.getAttribute("overlay") + self.store_version(package, cpv, slot, overlay) + + def store_package(self, cat, pkg, homepage, description): + created = False + obj = self.cache_get_package(cat, pkg) + + if not obj: + obj, created = Package.objects.get_or_create( + category=cat, + name=pkg, + homepage=homepage, + description=description, + ) + self.cache_store_package(obj) + + if created: + self.logger.info('+ [p] %s/%s' % (cat, pkg)) + + # Set all versions dead, then set found versions alive and + # delete old versions + if not self.kill_versions: + Version.objects.filter( + package=obj, + packaged=True + ).update(alive=False) + + return obj + + def store_version(self, package, cpv, slot, overlay): + cat, pkg, ver, rev = portage.catpkgsplit(cpv) + + overlays = self.overlays() + + if overlay in overlays: + overlay = overlays[overlay] + else: + overlay = 'gentoo' + + created = False + obj = self.cache_get_version( + package.category, package.name, ver, rev, slot, overlay + ) + if not obj: + obj, created = Version.objects.get_or_create( + package=package, slot=slot, + revision=rev, version=ver, + overlay=overlay, + defaults={ + "alive": True, + "packaged": True, + "version_type": get_version_type(ver), + "confidence": 100, + "handler": "portage" + } + ) + if not created: # Created objects have defaults values + obj.alive = True + obj.packaged = True + obj.save() + + if created: + self.cache_store_version(obj) + + # nothing to do (note: it can't be an upstream version because + # overlay can't be empty here) + if not created: + return + + self.logger.info('+ [v] %s' % (obj)) + + if overlay == 'gentoo': + package.n_packaged += 1 + else: + package.n_overlay += 1 + package.n_versions += 1 + package.save() + + if self.no_log: + return + + VersionLog.objects.create( + package=obj.package, + action=VersionLog.VERSION_ADDED, + slot=obj.slot, + revision=obj.revision, + version=obj.version, + overlay=obj.overlay + ) + + +@commit_on_success +def do_purge_versions(logger=None, no_log=False): + logger = logger or FakeLogger() + + # For each dead versions + for version in Version.objects.filter(packaged=True, alive=False): + if version.overlay == 'gentoo': + version.package.n_packaged -= 1 + else: + version.package.n_overlay -= 1 + version.package.n_versions -= 1 + version.package.save() + + logger.info('- [v] %s' % (version)) + + if no_log: + continue + + VersionLog.objects.create( + package=version.package, + action=VersionLog.VERSION_REMOVED, + slot=version.slot, + revision=version.revision, + version=version.version, + overlay=version.overlay + ) + + Version.objects.filter(packaged=True, alive=False).delete() + + +def scan_portage(packages=None, no_log=False, purge_packages=False, + purge_versions=False, prefetch=False, logger=None): + + logger = logger or FakeLogger() + kill_versions = False + if packages is None: + prefetch = True + kill_versions = True + + scan_handler = ScanPortage( + logger=logger, + no_log=no_log, + purge_packages=purge_packages, + kill_versions=kill_versions, + ) + + logger.info('Scanning portage tree...') + + if prefetch: + logger.info('Prefetching objects...') + for package in Package.objects.all(): + scan_handler.cache_store_package(package) + for version in Version.objects.select_related('package').all(): + scan_handler.cache_store_version(version) + logger.info('done') + + if not packages: + scan_handler.scan() + else: + for pkg in packages: + if isinstance(pkg, Package): + scan_handler.scan('%s/%s' % (pkg.category, pkg.name)) + else: + scan_handler.scan(pkg) + + if purge_versions: + do_purge_versions(logger=logger, no_log=no_log) + logger.info('Done.') + return True diff --git a/euscanwww/djeuscan/processing/scan_upstream.py b/euscanwww/djeuscan/processing/scan_upstream.py new file mode 100644 index 0000000..981af59 --- /dev/null +++ b/euscanwww/djeuscan/processing/scan_upstream.py @@ -0,0 +1,161 @@ +import portage + +from django.utils import timezone +from django.db.transaction import commit_on_success + +from euscan import CONFIG, output +from euscan.scan import scan_upstream as euscan_scan_upstream + +from djeuscan.processing import FakeLogger +from djeuscan.models import Package, Version, EuscanResult, VersionLog + + +class ScanUpstream(object): + def __init__(self, logger=None): + self.logger = logger or FakeLogger() + + def scan(self, package): + CONFIG["format"] = "dict" + output.clean() + output.set_query(package) + + euscan_scan_upstream(package) + + out = output.get_formatted_output() + out_json = output.get_formatted_output("json") + + try: + cpv = out[package]["metadata"]["cpv"] + scan_time = out[package]["metadata"]["scan_time"] + ebuild = out[package]["metadata"]["ebuild"] + except KeyError: + return {} + + with commit_on_success(): + obj = self.store_package(cpv) + + for res in out[package]["result"]: + self.store_version( + obj, + res["version"], + " ".join(res["urls"]), + res["type"], + res["handler"], + res["confidence"], + ) + + self.store_result(obj, out_json, scan_time, ebuild) + + return out + + def store_result(self, package, formatted_log, scan_time, ebuild): + # Remove previous logs + EuscanResult.objects.filter(package=package).delete() + + obj = EuscanResult() + obj.package = package + obj.result = formatted_log + obj.datetime = timezone.now() + obj.scan_time = scan_time + obj.ebuild = ebuild + obj.save() + + def store_package(self, cpv): + cat, pkg, ver, rev = portage.catpkgsplit(cpv) + + obj, created = Package.objects.get_or_create(category=cat, name=pkg) + + if created: + self.logger.info('+ [p] %s/%s' % (cat, pkg)) + + # Set all versions dead, then set found versions alive and + # delete old versions + Version.objects.filter(package=obj, packaged=False).update(alive=False) + + return obj + + def store_version(self, package, ver, url, version_type, handler, + confidence): + obj, created = Version.objects.get_or_create( + package=package, + slot='', + revision='r0', + version=ver, + overlay='', + defaults={"alive": True, "urls": url, "packaged": False, + "version_type": version_type, "handler": handler, + "confidence": confidence} + ) + if not created: + obj.alive = True + obj.urls = url + obj.packaged = False + obj.save() + + # If it's not a new version, just update the object and continue + if not created: + return + + self.logger.info('+ [u] %s %s' % (obj, url)) + + VersionLog.objects.create( + package=package, + action=VersionLog.VERSION_ADDED, + slot='', + revision='r0', + version=ver, + overlay='' + ) + + package.n_versions += 1 + package.save() + + +@commit_on_success +def do_purge_versions(logger=None): + logger = logger or FakeLogger() + + # For each dead versions + for version in Version.objects.filter(packaged=False, alive=False): + VersionLog.objects.create( + package=version.package, + action=VersionLog.VERSION_REMOVED, + slot=version.slot, + revision=version.revision, + version=version.version, + overlay=version.overlay + ) + + version.package.n_versions -= 1 + version.package.save() + + logger.info('- [u] %s %s' % (version, version.urls)) + Version.objects.filter(packaged=False, alive=False).delete() + + +def scan_upstream(packages=None, purge_versions=False, + logger=None): + logger = logger or FakeLogger() + + scan_handler = ScanUpstream(logger=logger) + + logger.info('Scanning upstream...') + + if not packages: + packages = Package.objects.all() + + result = True + + for pkg in packages: + if isinstance(pkg, Package): + curr = scan_handler.scan('%s/%s' % (pkg.category, pkg.name)) + else: + curr = scan_handler.scan(pkg) + if not curr: + result = False + + if purge_versions: + do_purge_versions(logger=logger) + + logger.info('Done.') + return result diff --git a/euscanwww/djeuscan/processing/update_counters.py b/euscanwww/djeuscan/processing/update_counters.py new file mode 100644 index 0000000..54cafed --- /dev/null +++ b/euscanwww/djeuscan/processing/update_counters.py @@ -0,0 +1,175 @@ +from django.db.transaction import commit_on_success +from django.utils import timezone + +from djeuscan.models import Package, Herd, Maintainer, Version +from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog +from djeuscan import charts +from djeuscan.processing import FakeLogger + +from distutils.version import StrictVersion, LooseVersion + + +def _compare_versions(version1, version2): + try: + return cmp(StrictVersion(version1), StrictVersion(version2)) + # in case of abnormal version number, fall back to LooseVersion + except ValueError: + return cmp(LooseVersion(version1), LooseVersion(version2)) + + +def _add_safe(storage, key): + if key not in storage: + storage[key] = 1 + else: + storage[key] += 1 + + +def _add_last_ver(storage, version): + key = version['package_id'] + if key not in storage: + storage[key] = version + return + if version['version'].startswith('9999'): + return + if _compare_versions(storage[key]['version'], + version['version']) < 0: + storage[key] = version + + +@commit_on_success +def update_counters(fast=False, nolog=False, logger=None): + logger = logger or FakeLogger() + + now = timezone.now() + + categories = {} + herds = {} + maintainers = {} + + wlog = None + + if not nolog: + wlog = WorldLog() + wlog.datetime = now + + for cat in Package.objects.values('category').distinct(): + clog = CategoryLog() + clog.datetime = now + clog.category = cat['category'] + categories[clog.category] = clog + + for herd in Herd.objects.all(): + hlog = HerdLog() + hlog.datetime = now + hlog.herd = herd + herds[herd.id] = hlog + + for maintainer in Maintainer.objects.all(): + mlog = MaintainerLog() + mlog.datetime = now + mlog.maintainer = maintainer + maintainers[maintainer.id] = mlog + + package_queryset = Package.objects.all() + + n_versions = {} + n_packaged = {} + n_overlay = {} + + last_versions_gentoo = {} + last_versions_overlay = {} + last_versions_upstream = {} + + if not fast: + attrs = ['id', 'version', 'overlay', 'packaged', 'package_id'] + for version in Version.objects.all().values(*attrs): + overlay, packaged = version['overlay'], version['packaged'] + package_id = version['package_id'] + + _add_safe(n_versions, package_id) + + if not packaged: + _add_last_ver(last_versions_upstream, version) + continue + if overlay == 'gentoo': + _add_safe(n_packaged, package_id) + _add_last_ver(last_versions_gentoo, version) + else: + _add_safe(n_overlay, package_id) + _add_last_ver(last_versions_overlay, version) + + for package in package_queryset.select_related('herds', 'maintainers'): + if not fast: + package.n_versions = n_versions.get(package.id, 0) + package.n_packaged = n_packaged.get(package.id, 0) + package.n_overlay = n_overlay.get(package.id, 0) + + default = {'id': None} + package.last_version_gentoo_id = last_versions_gentoo.get( + package.id, default + )['id'] + package.last_version_overlay_id = last_versions_overlay.get( + package.id, default + )['id'] + package.last_version_upstream_id = last_versions_upstream.get( + package.id, default + )['id'] + + package.save() + + n_packages_gentoo = int(package.n_packaged == package.n_versions) + n_packages_overlay = int(package.n_overlay and package.n_packaged \ + + package.n_overlay == package.n_versions) + n_packages_outdated = int(package.n_packaged + package.n_overlay \ + < package.n_versions) + + def update_row(storage, key): + storage[key].n_packages_gentoo += n_packages_gentoo + storage[key].n_packages_overlay += n_packages_overlay + storage[key].n_packages_outdated += n_packages_outdated + + storage[key].n_versions_gentoo += package.n_packaged + storage[key].n_versions_overlay += package.n_overlay + storage[key].n_versions_upstream += package.n_versions - \ + package.n_packaged - \ + package.n_overlay + + def update_log(storage, qs): + for row in qs: + update_row(storage, row['id']) + + if not nolog: + update_log(herds, package.herds.all().values('id')) + update_log(maintainers, package.maintainers.all().values('id')) + update_row(categories, package.category) + + wlog.n_packages_gentoo += n_packages_gentoo + wlog.n_packages_overlay += n_packages_overlay + wlog.n_packages_outdated += n_packages_outdated + + wlog.n_versions_gentoo += package.n_packaged + wlog.n_versions_overlay += package.n_overlay + wlog.n_versions_upstream += package.n_versions - \ + package.n_packaged - \ + package.n_overlay + + if nolog: + return + + for clog in categories.values(): + logger.info('+ [cl] %s\n' % clog) + charts.rrd_update('category-%s' % clog.category, now, clog) + clog.save() + + for hlog in herds.values(): + logger.info('+ [hl] %s\n' % hlog) + charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog) + hlog.save() + + for mlog in maintainers.values(): + logger.info('+ [ml] %s\n' % mlog) + charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog) + mlog.save() + + charts.rrd_update('world', now, wlog) + wlog.save() diff --git a/euscanwww/djeuscan/processing/update_portage_trees.py b/euscanwww/djeuscan/processing/update_portage_trees.py new file mode 100644 index 0000000..cbf0a0b --- /dev/null +++ b/euscanwww/djeuscan/processing/update_portage_trees.py @@ -0,0 +1,66 @@ +import subprocess +from StringIO import StringIO + +from django.conf import settings + +from djeuscan.processing import FakeLogger + + +def _launch_command(cmd): + """ + Helper for launching shell commands inside tasks + """ + fp = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output = StringIO(fp.communicate()[0]) + return output.getvalue() + + +def emerge_sync(): + """ + Launches an emerge --sync + """ + cmd = ["emerge", "--sync", "--root", settings.PORTAGE_ROOT, + "--config-root", settings.PORTAGE_CONFIGROOT] + return _launch_command(cmd) + + +def layman_sync(): + """ + Syncs Layman repos + """ + from layman import Layman + l = Layman(config=settings.LAYMAN_CONFIG) + return l.sync(l.get_installed(), output_results=False) + + +def emerge_regen(): + """ + Launches emerge --regen + """ + cmd = [ + "emerge", "--regen", "--jobs", settings.EMERGE_REGEN_JOBS, "--root", + settings.PORTAGE_ROOT, "--config-root", settings.PORTAGE_CONFIGROOT + ] + return _launch_command(cmd) + + +def eix_update(): + """ + Launches eix-update + """ + cmd = ["eix-update"] + return _launch_command(cmd) + + +def update_portage_trees(logger=None): + logger = logger or FakeLogger() + logger.info("Running emerge --sync") + emerge_sync() + logger.info("Running layman --sync") + layman_sync() + logger.info("Running emerge --regen") + emerge_regen() + logger.info("Running eix-update") + eix_update() + logger.info("Done!") diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py new file mode 100644 index 0000000..3fcbcae --- /dev/null +++ b/euscanwww/djeuscan/tasks.py @@ -0,0 +1,292 @@ +""" +Celery tasks for djeuscan +""" + +from itertools import islice + +from celery.task import task, periodic_task +from celery.task.schedules import crontab +from celery.task.sets import TaskSet + +from django.conf import settings + +from djeuscan.models import Package, RefreshPackageQuery +from djeuscan.processing.regen_rrds import regen_rrds +from djeuscan.processing.update_counters import update_counters +from djeuscan.processing.scan_metadata import scan_metadata +from djeuscan.processing.scan_portage import scan_portage +from djeuscan.processing.scan_upstream import scan_upstream +from djeuscan.processing.update_portage_trees import update_portage_trees + + +class TaskFailedException(Exception): + """ + Exception for failed tasks + """ + pass + + +def _chunks(it, n): + """ + Chunk generator, takes an iterator and the desired size of the chunk + """ + for first in it: + yield [first] + list(islice(it, n - 1)) + + +def _run_in_chunks(task, packages, kwargs=None, + concurrently=settings.TASKS_CONCURRENTLY, + n=settings.TASKS_SUBTASK_PACKAGES): + """ + Launches a TaskSet at a time with subtasks. + Each subtask has packages to handle + """ + output = [] + + chunk_generator = _chunks(iter(packages), n) + done = False + + while not done: + tasks = [] + for _ in range(concurrently): + try: + chunk = chunk_generator.next() + except StopIteration: + done = True + else: + tasks.append(task.subtask((chunk, ), kwargs)) + job = TaskSet(tasks=tasks) + result = job.apply_async() + # TODO: understand why this causes timeout + output.extend(list(result.join())) + return output + + +@task +def regen_rrds_task(): + """ + Regenerate RRDs + """ + return regen_rrds() + + +@task +def update_counters_task(fast=True): + """ + Updates counters + """ + return update_counters(fast=fast) + + +@task +def _scan_metadata_task(packages): + """ + Scans metadata for the given set of packages + """ + logger = _scan_metadata_task.get_logger() + logger.info("Starting metadata scanning subtask for %d packages...", + len(packages)) + + result = scan_metadata( + packages=packages, + logger=logger, + ) + if not result: + raise TaskFailedException + return result + + +@task +def scan_metadata_list_task(query): + """ + Runs a parallel metadata scan for packages in the query list (space + separated string). Task used only from the web interface. + """ + return _run_in_chunks(_scan_metadata_task, [p for p in query.split()]) + + +@task +def scan_metadata_all_task(): + """ + Runs a parallel metadata scan for all packages + """ + return _run_in_chunks(_scan_metadata_task, Package.objects.all()) + + +@task +def _scan_portage_task(packages, no_log=False, purge_packages=False, + purge_versions=False, prefetch=False): + """ + Scans portage for the given set of packages + """ + logger = _scan_portage_task.get_logger() + if packages: + logger.info("Starting portage scanning subtask for %d packages...", + len(packages)) + else: + logger.info("Starting portage scanning for all packages...") + + result = scan_portage( + packages=packages, + no_log=no_log, + purge_packages=purge_packages, + purge_versions=purge_versions, + prefetch=prefetch, + logger=logger, + ) + if not result: + raise TaskFailedException + return result + + +@task +def scan_portage_list_task(query, no_log=False, purge_packages=False, + purge_versions=False, prefetch=False): + """ + Runs a parallel portage scan for packages in the query list (space + separated string). Task used only from the web interface. + """ + kwargs = {"no_log": no_log, "purge_packages": purge_packages, + "purge_versions": purge_versions, "prefetch": prefetch} + return _run_in_chunks(_scan_portage_task, [p for p in query.split()], + kwargs) + + +@task +def scan_portage_all_task(no_log=False, purge_packages=False, + purge_versions=False, prefetch=False): + """ + Runs a syncronous portage scan for all packages + """ + return _scan_portage_task( + packages=None, + no_log=no_log, + purge_packages=purge_packages, + purge_versions=purge_versions, + prefetch=prefetch, + ) + + +@task +def _scan_upstream_task(packages, purge_versions=False): + """ + Scans upstream for the given set of packages + """ + logger = _scan_upstream_task.get_logger() + + logger.info("Starting upstream scanning subtask for %d packages...", + len(packages)) + + result = scan_upstream( + packages=packages, + purge_versions=purge_versions, + logger=logger, + ) + if not result: + raise TaskFailedException + return result + + +@task +def scan_upstream_list_task(query, purge_versions=False): + """ + Runs a parallel upstream scan for packages in the query list (space + separated string). Task used only from the web interface. + """ + + kwargs = {"purge_versions": purge_versions} + return _run_in_chunks(_scan_upstream_task, [p for p in query.split()], + kwargs) + + +@task +def scan_upstream_all_task(purge_versions=False): + """ + Runs a parallel portage scan for all packages + """ + kwargs = {"purge_versions": purge_versions} + return _run_in_chunks(_scan_upstream_task, Package.objects.all(), kwargs) + + +@task +def update_portage_trees_task(): + """ + Update portage tree + """ + logger = update_portage_trees_task.get_logger() + update_portage_trees(logger=logger) + + +@task +def update_task(update_portage_trees=True, scan_portage=True, + scan_metadata=True, scan_upstream=True, update_counter=True): + """ + Update the whole euscan system + """ + if update_portage_trees: + update_portage_trees_task() + if scan_portage: + scan_portage_all_task(prefetch=True, purge_packages=True, + purge_versions=True) + + # metadata and upstream scan can run concurrently, launch them + # asynchronously and wait for them to finish + metadata_job = None + if scan_metadata: + metadata_job = scan_metadata_all_task().delay() + + upstream_job = None + if scan_upstream: + upstream_job = scan_upstream_all_task().delay() + + if metadata_job: + metadata_job.wait() + if upstream_job: + upstream_job.wait() + + update_counters(fast=False) + + +@task +def scan_package_task(package): + _scan_portage_task([package], purge_packages=True, purge_versions=True) + _scan_metadata_task([package]) + _scan_upstream_task([package]) + + +@periodic_task(run_every=crontab(minute="*/1")) +def consume_refresh_package_request(): + """ + Satisfies user requests for package refreshing, runs every minute + """ + try: + obj = RefreshPackageQuery.objects.all().order_by('-priority')[0] + except IndexError: + return {} + else: + result = scan_package_task(obj.query) + obj.delete() + return result + + +@periodic_task(run_every=crontab(hour=03, minute=00, day_of_week=1)) +def update_periodic_task(): + """ + Runs a whole update once a week + """ + update_task() + + +admin_tasks = [ + regen_rrds_task, + update_counters_task, + scan_metadata_list_task, + scan_metadata_all_task, + scan_portage_all_task, + scan_portage_list_task, + scan_upstream_all_task, + scan_upstream_list_task, + update_portage_trees_task, + update_task, + scan_package_task, +] diff --git a/euscanwww/djeuscan/templates/_base.html b/euscanwww/djeuscan/templates/_base.html index e2bcd44..717eb20 100644 --- a/euscanwww/djeuscan/templates/_base.html +++ b/euscanwww/djeuscan/templates/_base.html @@ -10,6 +10,7 @@ {% endblock %} {% block css %} + {% endblock %} {% block javascript %} @@ -22,40 +23,65 @@ {% block header %}

Ebuild Upstream Scanner (euscan)

{% endblock %} -
- {% block content %}{% endblock %} -
-