euscanwww: use logging in djeuscan.processing

Signed-off-by: volpino <fox91@anche.no>

euscanwww/djeuscan/management/commands/regen_rrds.py

@@ -1,10 +1,16 @@
+import logging
 from django.core.management.base import BaseCommand
+
+from djeuscan.processing import set_verbosity_level
 from djeuscan.processing.regen_rrds import regen_rrds
 
+logger = logging.getLogger(__name__)
+
 
 class Command(BaseCommand):
     _overlays = {}
     help = 'Regenerate rrd database'
 
     def handle(self, *args, **options):
-        regen_rrds()
+        set_verbosity_level(logger, options.get("verbosity", 1))
+        regen_rrds(logger=logger)

euscanwww/djeuscan/management/commands/scan_metadata.py

@@ -1,10 +1,14 @@
 import sys
+import logging
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
 
+from djeuscan.processing import set_verbosity_level
 from djeuscan.processing.scan_metadata import scan_metadata
 
+logger = logging.getLogger(__name__)
+
 
 class Command(BaseCommand):
     _overlays = {}
@@ -15,16 +19,13 @@ class Command(BaseCommand):
             dest='all',
             default=False,
             help='Scan all packages'),
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
         )
     args = '<package package ...>'
     help = 'Scans metadata and fills database'
 
     def handle(self, *args, **options):
+        set_verbosity_level(logger, options.get("verbosity", 1))
+
         if options['all']:
             packages = None
 
@@ -33,4 +34,4 @@ class Command(BaseCommand):
         else:
             packages = [pkg[:-1] for pkg in sys.stdin.readlines()]
 
-        scan_metadata(packages=packages, quiet=options["quiet"])
+        scan_metadata(packages=packages, logger=logger)

euscanwww/djeuscan/management/commands/scan_portage.py

@@ -1,9 +1,14 @@
 import sys
+import logging
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
+
+from djeuscan.processing import set_verbosity_level
 from djeuscan.processing.scan_portage import scan_portage
 
+logger = logging.getLogger(__name__)
+
 
 class Command(BaseCommand):
     _overlays = {}
@@ -35,16 +40,13 @@ class Command(BaseCommand):
             default=False,
             help=('Prefetch all versions and packages from DB to '
                   'speedup full scan process.')),
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
         )
     args = '[package package ...]'
     help = 'Scans portage tree and fills database'
 
     def handle(self, *args, **options):
+        set_verbosity_level(logger, options.get("verbosity", 1))
+
         if options['all']:
             packages = None
 
@@ -59,6 +61,5 @@ class Command(BaseCommand):
             purge_packages=options["purge-packages"],
             purge_versions=options["purge-versions"],
             prefetch=options["prefetch"],
-            quiet=options["quiet"],
+            logger=logger,
-            stdout=self.stdout,
         )

euscanwww/djeuscan/management/commands/scan_upstream.py

@@ -1,10 +1,14 @@
+import logging
 import sys
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
 
+from djeuscan.processing import set_verbosity_level
 from djeuscan.processing.scan_upstream import scan_upstream
 
+logger = logging.getLogger(__name__)
+
 
 class Command(BaseCommand):
     _overlays = {}
@@ -20,16 +24,13 @@ class Command(BaseCommand):
             dest='purge-versions',
             default=False,
             help='Purge old versions'),
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
         )
     args = '<package package ...>'
     help = 'Scans metadata and fills database'
 
     def handle(self, *args, **options):
+        set_verbosity_level(logger, options.get("verbosity", 1))
+
         if options['all']:
             packages = None
 
@@ -41,6 +42,5 @@ class Command(BaseCommand):
         scan_upstream(
             packages=packages,
             purge_versions=options["purge-versions"],
-            quiet=options["quiet"],
+            logger=logger,
-            stdout=self.stdout,
         )

euscanwww/djeuscan/management/commands/update_counters.py

@@ -1,20 +1,19 @@
+import logging
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
 
+from djeuscan.processing import set_verbosity_level
 from djeuscan.processing.update_counters import update_counters
 
+logger = logging.getLogger(__name__)
+
 
 class Command(BaseCommand):
     _overlays = {}
     help = 'Update counters'
 
     option_list = BaseCommand.option_list + (
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
         make_option('--fast',
             action='store_true',
             dest='fast',
@@ -28,9 +27,9 @@ class Command(BaseCommand):
     )
 
     def handle(self, *args, **options):
+        set_verbosity_level(logger, options.get("verbosity", 1))
         update_counters(
-            stdout=self.stdout,
             fast=options["fast"],
-            quiet=options["quiet"],
             nolog=options["nolog"],
+            logger=logger,
         )

euscanwww/djeuscan/processing/__init__.py

@@ -1,3 +1,31 @@
+import logging
+
+
 class FakeLogger(object):
     def __getattr__(self, key):
         return lambda *x, **y: None
+
+
+def set_verbosity_level(logger, verbosity):
+    try:
+        verbosity = int(verbosity)
+    except (ValueError, TypeError):
+        return logger
+
+    levels = {
+        0: logging.DEBUG,
+        1: logging.INFO,
+        2: logging.WARNING,
+        3: logging.ERROR,
+        4: logging.CRITICAL
+    }
+
+    if verbosity < 0:
+        verbosity = 0
+
+    if verbosity > 4:
+        verbosity = 4
+
+    logger.setLevel(levels[verbosity])
+
+    return logger

euscanwww/djeuscan/processing/scan_metadata.py

@@ -1,4 +1,3 @@
-import sys
 from gentoolkit.query import Query
 from gentoolkit.errors import GentoolkitFatalError
 
@@ -7,12 +6,13 @@ from django.core.management.color import color_style
 from django.core.exceptions import ValidationError
 
 from djeuscan.models import Package, Herd, Maintainer
+from djeuscan.processing import FakeLogger
 
 
 class ScanMetadata(object):
-    def __init__(self, quiet=False):
+    def __init__(self, logger=None):
-        self.quiet = quiet
         self.style = color_style()
+        self.logger = logger or FakeLogger()
 
     @commit_on_success
     def scan(self, query=None, obj=None):
@@ -22,8 +22,8 @@ class ScanMetadata(object):
         )
 
         if not matches:
-            sys.stderr.write(
+            self.logger.error(
-                self.style.ERROR("Unknown package '%s'\n" % query)
+                self.style.ERROR("Unknown package '%s'" % query)
             )
             return
 
@@ -43,14 +43,14 @@ class ScanMetadata(object):
             obj.homepage = pkg.environment("HOMEPAGE")
             obj.description = pkg.environment("DESCRIPTION")
         except GentoolkitFatalError, err:
-            sys.stderr.write(
+            self.logger.error(
                 self.style.ERROR(
-                    "Gentoolkit fatal error: '%s'\n" % str(err)
+                    "Gentoolkit fatal error: '%s'" % str(err)
                 )
             )
 
-        if created and not self.quiet:
+        if created:
-            sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
+            self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name))
 
         if pkg.metadata:
             herds = dict(
@@ -92,8 +92,8 @@ class ScanMetadata(object):
                     )
                     obj.maintainers.add(maintainer)
                 except ValidationError:
-                    sys.stderr.write(
+                    self.logger.error(
-                        self.style.ERROR("Bad maintainer: '%s' '%s'\n" % \
+                        self.style.ERROR("Bad maintainer: '%s' '%s'" % \
                                          (maintainer.name, maintainer.email))
                     )
 
@@ -109,8 +109,8 @@ class ScanMetadata(object):
             defaults={"email": email}
         )
 
-        if created and not self.quiet:
+        if created:
-            sys.stdout.write('+ [h] %s <%s>\n' % (name, email))
+            self.logger.info('+ [h] %s <%s>' % (name, email))
 
         herd.email = email
         herd.save()
@@ -129,15 +129,14 @@ class ScanMetadata(object):
         )
 
         if created:
-            if not self.quiet:
+            self.logger.info(
-                sys.stdout.write(
+                '+ [m] %s <%s>' % (name.encode('utf-8'), email)
-                    '+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
+            )
-                )
         return maintainer
 
 
-def scan_metadata(packages=None, quiet=False, logger=None):
+def scan_metadata(packages=None, logger=None):
-    scan_handler = ScanMetadata(quiet=quiet)
+    scan_handler = ScanMetadata(logger=logger)
     if packages is None:
         packages = Package.objects.all()
 

euscanwww/djeuscan/processing/scan_portage.py

@@ -1,24 +1,23 @@
 import subprocess
 import portage
-import sys
 import os
 import re
 
 from django.db.transaction import commit_on_success
 from django.core.management.color import color_style
 
+from djeuscan.processing import FakeLogger
 from djeuscan.models import Package, Version, VersionLog
 
 
 class ScanPortage(object):
-    def __init__(self, stdout=None, no_log=False, purge_packages=False,
+    def __init__(self, logger=None, no_log=False, purge_packages=False,
-                 purge_versions=False, kill_versions=False, quiet=False):
+                 purge_versions=False, kill_versions=False):
-        self.stdout = sys.stdout if stdout is None else stdout
+        self.logger = logger or FakeLogger()
         self.no_log = no_log
         self.purge_packages = purge_packages
         self.purge_versions = purge_versions
         self.kill_versions = kill_versions
-        self.quiet = quiet
 
         self.style = color_style()
         self._cache = {'packages': {}, 'versions': {}}
@@ -93,12 +92,9 @@ class ScanPortage(object):
             cmd.extend(['--exact', query])
 
         if self.kill_versions:
-            if not self.quiet:
+            self.logger.info('Killing existing versions...')
-                self.stdout.write('Killing existing versions...')
-                self.stdout.flush()
             Version.objects.filter(packaged=True).update(alive=False)
-            if not self.quiet:
+            self.logger.info('done')
-                self.stdout.write('done\n')
 
         output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
             communicate()[0]
@@ -108,17 +104,16 @@ class ScanPortage(object):
             if not query:
                 return
             if self.purge_packages:
-                if not self.quiet:
+                self.logger.info('- [p] %s' % (query))
-                    sys.stdout.write('- [p] %s\n' % (query))
                 if '/' in query:
                     cat, pkg = portage.catsplit(query)
                     Package.objects.filter(category=cat, name=pkg).delete()
                 else:
                     Package.objects.filter(name=query).delete()
             else:
-                sys.stderr.write(
+                self.logger.error(
                     self.style.ERROR(
-                        "Unknown package '%s'\n" % query
+                        "Unknown package '%s'" % query
                     )
                 )
             return
@@ -156,8 +151,7 @@ class ScanPortage(object):
             for package in Package.objects.all():
                 cp = "%s/%s" % (package.category, package.name)
                 if cp not in packages:
-                    if not self.quiet:
+                    self.logger.info('- [p] %s' % (package))
-                        sys.stdout.write('- [p] %s\n' % (package))
                     package.delete()
 
     def store_package(self, cat, pkg):
@@ -172,8 +166,7 @@ class ScanPortage(object):
             self.cache_store_package(obj)
 
         if created:
-            if not self.quiet:
+            self.logger.info('+ [p] %s/%s' % (cat, pkg))
-                sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
 
         # Set all versions dead, then set found versions alive and
         # delete old versions
@@ -219,8 +212,7 @@ class ScanPortage(object):
         if not created:
             return
 
-        if not self.quiet:
+        self.logger.info('+ [v] %s' % (obj))
-            sys.stdout.write('+ [v] %s \n' % (obj))
 
         if overlay == 'gentoo':
             package.n_packaged += 1
@@ -243,7 +235,9 @@ class ScanPortage(object):
 
 
 @commit_on_success
-def purge_versions(quiet=False, nolog=False):
+def purge_versions(logger=None, nolog=False):
+    logger = logger or FakeLogger()
+
     # For each dead versions
     for version in Version.objects.filter(packaged=True, alive=False):
         if version.overlay == 'gentoo':
@@ -253,8 +247,7 @@ def purge_versions(quiet=False, nolog=False):
         version.package.n_versions -= 1
         version.package.save()
 
-        if not quiet:
+        logger.info('- [v] %s' % (version))
-            sys.stdout.write('- [v] %s\n' % (version))
 
         if nolog:
             continue
@@ -272,37 +265,31 @@ def purge_versions(quiet=False, nolog=False):
 
 
 def scan_portage(packages=None, no_log=False, purge_packages=False,
-                 purge_versions=False, prefetch=False, logger=None,
+                 purge_versions=False, prefetch=False, logger=None):
-                 quiet=False, stdout=None):
-    stdout = sys.stdout if stdout is None else stdout
 
+    logger = logger or FakeLogger()
     kill_versions = False
     if packages is None:
         prefetch = True
         kill_versions = True
 
     scan_handler = ScanPortage(
-        stdout=stdout,
+        logger=logger,
         no_log=no_log,
         purge_packages=purge_packages,
         purge_versions=purge_versions,
         kill_versions=kill_versions,
-        quiet=quiet,
     )
 
-    if not quiet:
+    logger.info('Scanning portage tree...')
-        stdout.write('Scanning portage tree...\n')
 
     if prefetch:
-        if quiet:
+        logger.info('Prefetching objects...')
-            stdout.write('Prefetching objects...')
-            stdout.flush()
         for package in Package.objects.all():
             scan_handler.cache_store_package(package)
         for version in Version.objects.select_related('package').all():
             scan_handler.cache_store_version(version)
-        if quiet:
+        logger.info('done')
-            stdout.write('done\n')
 
     if packages is None:
         scan_handler.scan()
@@ -314,7 +301,6 @@ def scan_portage(packages=None, no_log=False, purge_packages=False,
                 scan_handler.scan(pkg)
 
     if purge_versions:
-        purge_versions(quiet, no_log)
+        purge_versions(logger=logger, no_log=no_log)
 
-    if not quiet:
+    logger.info('Done.')
-        stdout.write('Done.\n')

euscanwww/djeuscan/processing/scan_upstream.py

@@ -1,5 +1,4 @@
 import portage
-import sys
 
 from django.utils import timezone
 from django.db.transaction import commit_on_success
@@ -7,12 +6,13 @@ from django.db.transaction import commit_on_success
 from euscan import CONFIG, output
 from euscan.scan import scan_upstream as euscan_scan_upstream
 
+from djeuscan.processing import FakeLogger
 from djeuscan.models import Package, Version, EuscanResult, VersionLog
 
 
 class ScanUpstream(object):
-    def __init__(self, quiet=False):
+    def __init__(self, logger=None):
-        self.quiet = quiet
+        self.logger = logger or FakeLogger()
 
     def scan(self, package):
         CONFIG["format"] = "dict"
@@ -52,8 +52,8 @@ class ScanUpstream(object):
 
         obj, created = Package.objects.get_or_create(category=cat, name=pkg)
 
-        if created and not self.quiet:
+        if created:
-            sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
+            self.logger.info('+ [p] %s/%s' % (cat, pkg))
 
         # Set all versions dead, then set found versions alive and
         # delete old versions
@@ -76,8 +76,7 @@ class ScanUpstream(object):
         if not created:
             return
 
-        if not self.quiet:
+        self.logger.info('+ [u] %s %s' % (obj, url))
-            sys.stdout.write('+ [u] %s %s\n' % (obj, url))
 
         VersionLog.objects.create(
             package=package,
@@ -93,7 +92,9 @@ class ScanUpstream(object):
 
 
 @commit_on_success
-def purge_versions(quiet=False):
+def purge_versions(logger=None):
+    logger = logger or FakeLogger()
+
     # For each dead versions
     for version in Version.objects.filter(packaged=False, alive=False):
         VersionLog.objects.create(
@@ -108,20 +109,17 @@ def purge_versions(quiet=False):
         version.package.n_versions -= 1
         version.package.save()
 
-        if not quiet:
+        logger.info('- [u] %s %s' % (version, version.urls))
-            sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
     Version.objects.filter(packaged=False, alive=False).delete()
 
 
-def scan_upstream(packages=None, purge_versions=False, quiet=False,
+def scan_upstream(packages=None, purge_versions=False,
-                  logger=None, stdout=None):
+                  logger=None):
+    logger = logger or FakeLogger()
 
-    stdout = sys.stdout if stdout is None else stdout
+    scan_handler = ScanUpstream(logger=logger)
 
-    scan_handler = ScanUpstream(quiet)
+    logger.info('Scanning upstream...')
-
-    if not quiet:
-        stdout.write('Scanning upstream...\n')
 
     if packages is None:
         packages = Package.objects.all()
@@ -133,7 +131,6 @@ def scan_upstream(packages=None, purge_versions=False, quiet=False,
             scan_handler.scan(pkg)
 
     if purge_versions:
-        purge_versions(quiet)
+        purge_versions(logger=logger)
 
-    if not quiet:
+    logger.info('Done.')
-        stdout.write('Done.\n')

euscanwww/djeuscan/processing/update_counters.py

@@ -1,5 +1,3 @@
-import sys
-
 from django.db.transaction import commit_on_success
 from django.utils import timezone
 
@@ -39,13 +37,8 @@ def _add_last_ver(storage, version):
 
 
 @commit_on_success
-def update_counters(stdout=None, fast=False, quiet=False, nolog=False,
+def update_counters(fast=False, nolog=False, logger=None):
-                    logger=None):
+    logger = logger or FakeLogger()
-    if logger is None:
-        logger = FakeLogger()  # TODO: write log messages
-
-    if stdout is None:
-        stdout = sys.stdout
 
     now = timezone.now()
 
@@ -164,20 +157,17 @@ def update_counters(stdout=None, fast=False, quiet=False, nolog=False,
         return
 
     for clog in categories.values():
-        if not quiet:
+        logger.info('+ [cl] %s\n' % clog)
-            stdout.write('+ [cl] %s\n' % clog)
         charts.rrd_update('category-%s' % clog.category, now, clog)
         clog.save()
 
     for hlog in herds.values():
-        if not quiet:
+        logger.info('+ [hl] %s\n' % hlog)
-            stdout.write('+ [hl] %s\n' % hlog)
         charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
         hlog.save()
 
     for mlog in maintainers.values():
-        if not quiet:
+        logger.info('+ [ml] %s\n' % mlog)
-            stdout.write('+ [ml] %s\n' % mlog)
         charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
         mlog.save()
 

euscanwww/euscanwww/settings.py

@@ -181,12 +181,22 @@ INSTALLED_APPS = (
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
+    'formatters': {
+        'simple': {
+            'format': '%(levelname)s %(asctime)s %(message)s'
+        },
+    },
     'filters': {
         'require_debug_false': {
             '()': 'django.utils.log.RequireDebugFalse'
         }
     },
     'handlers': {
+        'console': {
+            'level': 'INFO',
+            'class': 'logging.StreamHandler',
+            'formatter': 'simple'
+        },
         'mail_admins': {
             'level': 'ERROR',
             'filters': ['require_debug_false'],
@@ -199,6 +209,11 @@ LOGGING = {
             'level': 'ERROR',
             'propagate': True,
         },
+        'djeuscan': {
+            'handlers': ['console'],
+            'level': 'INFO',
+            'propagate': True
+        }
     }
 }