diff --git a/euscanwww/djeuscan/management/commands/regen_rrds.py b/euscanwww/djeuscan/management/commands/regen_rrds.py index f6b2ba0..a53f44d 100644 --- a/euscanwww/djeuscan/management/commands/regen_rrds.py +++ b/euscanwww/djeuscan/management/commands/regen_rrds.py @@ -1,10 +1,16 @@ +import logging from django.core.management.base import BaseCommand + +from djeuscan.processing import set_verbosity_level from djeuscan.processing.regen_rrds import regen_rrds +logger = logging.getLogger(__name__) + class Command(BaseCommand): _overlays = {} help = 'Regenerate rrd database' def handle(self, *args, **options): - regen_rrds() + set_verbosity_level(logger, options.get("verbosity", 1)) + regen_rrds(logger=logger) diff --git a/euscanwww/djeuscan/management/commands/scan_metadata.py b/euscanwww/djeuscan/management/commands/scan_metadata.py index 5a19315..5decb70 100644 --- a/euscanwww/djeuscan/management/commands/scan_metadata.py +++ b/euscanwww/djeuscan/management/commands/scan_metadata.py @@ -1,10 +1,14 @@ import sys +import logging from optparse import make_option from django.core.management.base import BaseCommand +from djeuscan.processing import set_verbosity_level from djeuscan.processing.scan_metadata import scan_metadata +logger = logging.getLogger(__name__) + class Command(BaseCommand): _overlays = {} @@ -15,16 +19,13 @@ class Command(BaseCommand): dest='all', default=False, help='Scan all packages'), - make_option('--quiet', - action='store_true', - dest='quiet', - default=False, - help='Be quiet'), ) args = '' help = 'Scans metadata and fills database' def handle(self, *args, **options): + set_verbosity_level(logger, options.get("verbosity", 1)) + if options['all']: packages = None @@ -33,4 +34,4 @@ class Command(BaseCommand): else: packages = [pkg[:-1] for pkg in sys.stdin.readlines()] - scan_metadata(packages=packages, quiet=options["quiet"]) + scan_metadata(packages=packages, logger=logger) diff --git a/euscanwww/djeuscan/management/commands/scan_portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py index 2bfede8..c0dd6fe 100644 --- a/euscanwww/djeuscan/management/commands/scan_portage.py +++ b/euscanwww/djeuscan/management/commands/scan_portage.py @@ -1,9 +1,14 @@ import sys +import logging from optparse import make_option from django.core.management.base import BaseCommand + +from djeuscan.processing import set_verbosity_level from djeuscan.processing.scan_portage import scan_portage +logger = logging.getLogger(__name__) + class Command(BaseCommand): _overlays = {} @@ -35,16 +40,13 @@ class Command(BaseCommand): default=False, help=('Prefetch all versions and packages from DB to ' 'speedup full scan process.')), - make_option('--quiet', - action='store_true', - dest='quiet', - default=False, - help='Be quiet'), ) args = '[package package ...]' help = 'Scans portage tree and fills database' def handle(self, *args, **options): + set_verbosity_level(logger, options.get("verbosity", 1)) + if options['all']: packages = None @@ -59,6 +61,5 @@ class Command(BaseCommand): purge_packages=options["purge-packages"], purge_versions=options["purge-versions"], prefetch=options["prefetch"], - quiet=options["quiet"], - stdout=self.stdout, + logger=logger, ) diff --git a/euscanwww/djeuscan/management/commands/scan_upstream.py b/euscanwww/djeuscan/management/commands/scan_upstream.py index 043fb83..f185c38 100644 --- a/euscanwww/djeuscan/management/commands/scan_upstream.py +++ b/euscanwww/djeuscan/management/commands/scan_upstream.py @@ -1,10 +1,14 @@ +import logging import sys from optparse import make_option from django.core.management.base import BaseCommand +from djeuscan.processing import set_verbosity_level from djeuscan.processing.scan_upstream import scan_upstream +logger = logging.getLogger(__name__) + class Command(BaseCommand): _overlays = {} @@ -20,16 +24,13 @@ class Command(BaseCommand): dest='purge-versions', default=False, help='Purge old versions'), - make_option('--quiet', - action='store_true', - dest='quiet', - default=False, - help='Be quiet'), ) args = '' help = 'Scans metadata and fills database' def handle(self, *args, **options): + set_verbosity_level(logger, options.get("verbosity", 1)) + if options['all']: packages = None @@ -41,6 +42,5 @@ class Command(BaseCommand): scan_upstream( packages=packages, purge_versions=options["purge-versions"], - quiet=options["quiet"], - stdout=self.stdout, + logger=logger, ) diff --git a/euscanwww/djeuscan/management/commands/update_counters.py b/euscanwww/djeuscan/management/commands/update_counters.py index 7dc407e..100d5a7 100644 --- a/euscanwww/djeuscan/management/commands/update_counters.py +++ b/euscanwww/djeuscan/management/commands/update_counters.py @@ -1,20 +1,19 @@ +import logging from optparse import make_option from django.core.management.base import BaseCommand +from djeuscan.processing import set_verbosity_level from djeuscan.processing.update_counters import update_counters +logger = logging.getLogger(__name__) + class Command(BaseCommand): _overlays = {} help = 'Update counters' option_list = BaseCommand.option_list + ( - make_option('--quiet', - action='store_true', - dest='quiet', - default=False, - help='Be quiet'), make_option('--fast', action='store_true', dest='fast', @@ -28,9 +27,9 @@ class Command(BaseCommand): ) def handle(self, *args, **options): + set_verbosity_level(logger, options.get("verbosity", 1)) update_counters( - stdout=self.stdout, fast=options["fast"], - quiet=options["quiet"], nolog=options["nolog"], + logger=logger, ) diff --git a/euscanwww/djeuscan/processing/__init__.py b/euscanwww/djeuscan/processing/__init__.py index c4ea508..4a203b7 100644 --- a/euscanwww/djeuscan/processing/__init__.py +++ b/euscanwww/djeuscan/processing/__init__.py @@ -1,3 +1,31 @@ +import logging + + class FakeLogger(object): def __getattr__(self, key): return lambda *x, **y: None + + +def set_verbosity_level(logger, verbosity): + try: + verbosity = int(verbosity) + except (ValueError, TypeError): + return logger + + levels = { + 0: logging.DEBUG, + 1: logging.INFO, + 2: logging.WARNING, + 3: logging.ERROR, + 4: logging.CRITICAL + } + + if verbosity < 0: + verbosity = 0 + + if verbosity > 4: + verbosity = 4 + + logger.setLevel(levels[verbosity]) + + return logger diff --git a/euscanwww/djeuscan/processing/scan_metadata.py b/euscanwww/djeuscan/processing/scan_metadata.py index 15ab108..4f06d05 100644 --- a/euscanwww/djeuscan/processing/scan_metadata.py +++ b/euscanwww/djeuscan/processing/scan_metadata.py @@ -1,4 +1,3 @@ -import sys from gentoolkit.query import Query from gentoolkit.errors import GentoolkitFatalError @@ -7,12 +6,13 @@ from django.core.management.color import color_style from django.core.exceptions import ValidationError from djeuscan.models import Package, Herd, Maintainer +from djeuscan.processing import FakeLogger class ScanMetadata(object): - def __init__(self, quiet=False): - self.quiet = quiet + def __init__(self, logger=None): self.style = color_style() + self.logger = logger or FakeLogger() @commit_on_success def scan(self, query=None, obj=None): @@ -22,8 +22,8 @@ class ScanMetadata(object): ) if not matches: - sys.stderr.write( - self.style.ERROR("Unknown package '%s'\n" % query) + self.logger.error( + self.style.ERROR("Unknown package '%s'" % query) ) return @@ -43,14 +43,14 @@ class ScanMetadata(object): obj.homepage = pkg.environment("HOMEPAGE") obj.description = pkg.environment("DESCRIPTION") except GentoolkitFatalError, err: - sys.stderr.write( + self.logger.error( self.style.ERROR( - "Gentoolkit fatal error: '%s'\n" % str(err) + "Gentoolkit fatal error: '%s'" % str(err) ) ) - if created and not self.quiet: - sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name)) + if created: + self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name)) if pkg.metadata: herds = dict( @@ -92,8 +92,8 @@ class ScanMetadata(object): ) obj.maintainers.add(maintainer) except ValidationError: - sys.stderr.write( - self.style.ERROR("Bad maintainer: '%s' '%s'\n" % \ + self.logger.error( + self.style.ERROR("Bad maintainer: '%s' '%s'" % \ (maintainer.name, maintainer.email)) ) @@ -109,8 +109,8 @@ class ScanMetadata(object): defaults={"email": email} ) - if created and not self.quiet: - sys.stdout.write('+ [h] %s <%s>\n' % (name, email)) + if created: + self.logger.info('+ [h] %s <%s>' % (name, email)) herd.email = email herd.save() @@ -129,15 +129,14 @@ class ScanMetadata(object): ) if created: - if not self.quiet: - sys.stdout.write( - '+ [m] %s <%s>\n' % (name.encode('utf-8'), email) - ) + self.logger.info( + '+ [m] %s <%s>' % (name.encode('utf-8'), email) + ) return maintainer -def scan_metadata(packages=None, quiet=False, logger=None): - scan_handler = ScanMetadata(quiet=quiet) +def scan_metadata(packages=None, logger=None): + scan_handler = ScanMetadata(logger=logger) if packages is None: packages = Package.objects.all() diff --git a/euscanwww/djeuscan/processing/scan_portage.py b/euscanwww/djeuscan/processing/scan_portage.py index 43e91f4..fdebc57 100644 --- a/euscanwww/djeuscan/processing/scan_portage.py +++ b/euscanwww/djeuscan/processing/scan_portage.py @@ -1,24 +1,23 @@ import subprocess import portage -import sys import os import re from django.db.transaction import commit_on_success from django.core.management.color import color_style +from djeuscan.processing import FakeLogger from djeuscan.models import Package, Version, VersionLog class ScanPortage(object): - def __init__(self, stdout=None, no_log=False, purge_packages=False, - purge_versions=False, kill_versions=False, quiet=False): - self.stdout = sys.stdout if stdout is None else stdout + def __init__(self, logger=None, no_log=False, purge_packages=False, + purge_versions=False, kill_versions=False): + self.logger = logger or FakeLogger() self.no_log = no_log self.purge_packages = purge_packages self.purge_versions = purge_versions self.kill_versions = kill_versions - self.quiet = quiet self.style = color_style() self._cache = {'packages': {}, 'versions': {}} @@ -93,12 +92,9 @@ class ScanPortage(object): cmd.extend(['--exact', query]) if self.kill_versions: - if not self.quiet: - self.stdout.write('Killing existing versions...') - self.stdout.flush() + self.logger.info('Killing existing versions...') Version.objects.filter(packaged=True).update(alive=False) - if not self.quiet: - self.stdout.write('done\n') + self.logger.info('done') output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\ communicate()[0] @@ -108,17 +104,16 @@ class ScanPortage(object): if not query: return if self.purge_packages: - if not self.quiet: - sys.stdout.write('- [p] %s\n' % (query)) + self.logger.info('- [p] %s' % (query)) if '/' in query: cat, pkg = portage.catsplit(query) Package.objects.filter(category=cat, name=pkg).delete() else: Package.objects.filter(name=query).delete() else: - sys.stderr.write( + self.logger.error( self.style.ERROR( - "Unknown package '%s'\n" % query + "Unknown package '%s'" % query ) ) return @@ -156,8 +151,7 @@ class ScanPortage(object): for package in Package.objects.all(): cp = "%s/%s" % (package.category, package.name) if cp not in packages: - if not self.quiet: - sys.stdout.write('- [p] %s\n' % (package)) + self.logger.info('- [p] %s' % (package)) package.delete() def store_package(self, cat, pkg): @@ -172,8 +166,7 @@ class ScanPortage(object): self.cache_store_package(obj) if created: - if not self.quiet: - sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg)) + self.logger.info('+ [p] %s/%s' % (cat, pkg)) # Set all versions dead, then set found versions alive and # delete old versions @@ -219,8 +212,7 @@ class ScanPortage(object): if not created: return - if not self.quiet: - sys.stdout.write('+ [v] %s \n' % (obj)) + self.logger.info('+ [v] %s' % (obj)) if overlay == 'gentoo': package.n_packaged += 1 @@ -243,7 +235,9 @@ class ScanPortage(object): @commit_on_success -def purge_versions(quiet=False, nolog=False): +def purge_versions(logger=None, nolog=False): + logger = logger or FakeLogger() + # For each dead versions for version in Version.objects.filter(packaged=True, alive=False): if version.overlay == 'gentoo': @@ -253,8 +247,7 @@ def purge_versions(quiet=False, nolog=False): version.package.n_versions -= 1 version.package.save() - if not quiet: - sys.stdout.write('- [v] %s\n' % (version)) + logger.info('- [v] %s' % (version)) if nolog: continue @@ -272,37 +265,31 @@ def purge_versions(quiet=False, nolog=False): def scan_portage(packages=None, no_log=False, purge_packages=False, - purge_versions=False, prefetch=False, logger=None, - quiet=False, stdout=None): - stdout = sys.stdout if stdout is None else stdout + purge_versions=False, prefetch=False, logger=None): + logger = logger or FakeLogger() kill_versions = False if packages is None: prefetch = True kill_versions = True scan_handler = ScanPortage( - stdout=stdout, + logger=logger, no_log=no_log, purge_packages=purge_packages, purge_versions=purge_versions, kill_versions=kill_versions, - quiet=quiet, ) - if not quiet: - stdout.write('Scanning portage tree...\n') + logger.info('Scanning portage tree...') if prefetch: - if quiet: - stdout.write('Prefetching objects...') - stdout.flush() + logger.info('Prefetching objects...') for package in Package.objects.all(): scan_handler.cache_store_package(package) for version in Version.objects.select_related('package').all(): scan_handler.cache_store_version(version) - if quiet: - stdout.write('done\n') + logger.info('done') if packages is None: scan_handler.scan() @@ -314,7 +301,6 @@ def scan_portage(packages=None, no_log=False, purge_packages=False, scan_handler.scan(pkg) if purge_versions: - purge_versions(quiet, no_log) + purge_versions(logger=logger, no_log=no_log) - if not quiet: - stdout.write('Done.\n') + logger.info('Done.') diff --git a/euscanwww/djeuscan/processing/scan_upstream.py b/euscanwww/djeuscan/processing/scan_upstream.py index 9eee48b..2791ce5 100644 --- a/euscanwww/djeuscan/processing/scan_upstream.py +++ b/euscanwww/djeuscan/processing/scan_upstream.py @@ -1,5 +1,4 @@ import portage -import sys from django.utils import timezone from django.db.transaction import commit_on_success @@ -7,12 +6,13 @@ from django.db.transaction import commit_on_success from euscan import CONFIG, output from euscan.scan import scan_upstream as euscan_scan_upstream +from djeuscan.processing import FakeLogger from djeuscan.models import Package, Version, EuscanResult, VersionLog class ScanUpstream(object): - def __init__(self, quiet=False): - self.quiet = quiet + def __init__(self, logger=None): + self.logger = logger or FakeLogger() def scan(self, package): CONFIG["format"] = "dict" @@ -52,8 +52,8 @@ class ScanUpstream(object): obj, created = Package.objects.get_or_create(category=cat, name=pkg) - if created and not self.quiet: - sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg)) + if created: + self.logger.info('+ [p] %s/%s' % (cat, pkg)) # Set all versions dead, then set found versions alive and # delete old versions @@ -76,8 +76,7 @@ class ScanUpstream(object): if not created: return - if not self.quiet: - sys.stdout.write('+ [u] %s %s\n' % (obj, url)) + self.logger.info('+ [u] %s %s' % (obj, url)) VersionLog.objects.create( package=package, @@ -93,7 +92,9 @@ class ScanUpstream(object): @commit_on_success -def purge_versions(quiet=False): +def purge_versions(logger=None): + logger = logger or FakeLogger() + # For each dead versions for version in Version.objects.filter(packaged=False, alive=False): VersionLog.objects.create( @@ -108,20 +109,17 @@ def purge_versions(quiet=False): version.package.n_versions -= 1 version.package.save() - if not quiet: - sys.stdout.write('- [u] %s %s\n' % (version, version.urls)) + logger.info('- [u] %s %s' % (version, version.urls)) Version.objects.filter(packaged=False, alive=False).delete() -def scan_upstream(packages=None, purge_versions=False, quiet=False, - logger=None, stdout=None): +def scan_upstream(packages=None, purge_versions=False, + logger=None): + logger = logger or FakeLogger() - stdout = sys.stdout if stdout is None else stdout + scan_handler = ScanUpstream(logger=logger) - scan_handler = ScanUpstream(quiet) - - if not quiet: - stdout.write('Scanning upstream...\n') + logger.info('Scanning upstream...') if packages is None: packages = Package.objects.all() @@ -133,7 +131,6 @@ def scan_upstream(packages=None, purge_versions=False, quiet=False, scan_handler.scan(pkg) if purge_versions: - purge_versions(quiet) + purge_versions(logger=logger) - if not quiet: - stdout.write('Done.\n') + logger.info('Done.') diff --git a/euscanwww/djeuscan/processing/update_counters.py b/euscanwww/djeuscan/processing/update_counters.py index 6d1c832..54cafed 100644 --- a/euscanwww/djeuscan/processing/update_counters.py +++ b/euscanwww/djeuscan/processing/update_counters.py @@ -1,5 +1,3 @@ -import sys - from django.db.transaction import commit_on_success from django.utils import timezone @@ -39,13 +37,8 @@ def _add_last_ver(storage, version): @commit_on_success -def update_counters(stdout=None, fast=False, quiet=False, nolog=False, - logger=None): - if logger is None: - logger = FakeLogger() # TODO: write log messages - - if stdout is None: - stdout = sys.stdout +def update_counters(fast=False, nolog=False, logger=None): + logger = logger or FakeLogger() now = timezone.now() @@ -164,20 +157,17 @@ def update_counters(stdout=None, fast=False, quiet=False, nolog=False, return for clog in categories.values(): - if not quiet: - stdout.write('+ [cl] %s\n' % clog) + logger.info('+ [cl] %s\n' % clog) charts.rrd_update('category-%s' % clog.category, now, clog) clog.save() for hlog in herds.values(): - if not quiet: - stdout.write('+ [hl] %s\n' % hlog) + logger.info('+ [hl] %s\n' % hlog) charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog) hlog.save() for mlog in maintainers.values(): - if not quiet: - stdout.write('+ [ml] %s\n' % mlog) + logger.info('+ [ml] %s\n' % mlog) charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog) mlog.save() diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py index 2e7628c..116fcd9 100644 --- a/euscanwww/euscanwww/settings.py +++ b/euscanwww/euscanwww/settings.py @@ -181,12 +181,22 @@ INSTALLED_APPS = ( LOGGING = { 'version': 1, 'disable_existing_loggers': False, + 'formatters': { + 'simple': { + 'format': '%(levelname)s %(asctime)s %(message)s' + }, + }, 'filters': { 'require_debug_false': { '()': 'django.utils.log.RequireDebugFalse' } }, 'handlers': { + 'console': { + 'level': 'INFO', + 'class': 'logging.StreamHandler', + 'formatter': 'simple' + }, 'mail_admins': { 'level': 'ERROR', 'filters': ['require_debug_false'], @@ -199,6 +209,11 @@ LOGGING = { 'level': 'ERROR', 'propagate': True, }, + 'djeuscan': { + 'handlers': ['console'], + 'level': 'INFO', + 'propagate': True + } } }