euscanwww: use logging in djeuscan.processing

Signed-off-by: volpino <fox91@anche.no>
This commit is contained in:
volpino 2012-06-22 11:31:15 +02:00
parent 9af5548178
commit 779ccdf5d3
11 changed files with 142 additions and 120 deletions

View File

@ -1,10 +1,16 @@
import logging
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from djeuscan.processing import set_verbosity_level
from djeuscan.processing.regen_rrds import regen_rrds from djeuscan.processing.regen_rrds import regen_rrds
logger = logging.getLogger(__name__)
class Command(BaseCommand): class Command(BaseCommand):
_overlays = {} _overlays = {}
help = 'Regenerate rrd database' help = 'Regenerate rrd database'
def handle(self, *args, **options): def handle(self, *args, **options):
regen_rrds() set_verbosity_level(logger, options.get("verbosity", 1))
regen_rrds(logger=logger)

View File

@ -1,10 +1,14 @@
import sys import sys
import logging
from optparse import make_option from optparse import make_option
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from djeuscan.processing import set_verbosity_level
from djeuscan.processing.scan_metadata import scan_metadata from djeuscan.processing.scan_metadata import scan_metadata
logger = logging.getLogger(__name__)
class Command(BaseCommand): class Command(BaseCommand):
_overlays = {} _overlays = {}
@ -15,16 +19,13 @@ class Command(BaseCommand):
dest='all', dest='all',
default=False, default=False,
help='Scan all packages'), help='Scan all packages'),
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
) )
args = '<package package ...>' args = '<package package ...>'
help = 'Scans metadata and fills database' help = 'Scans metadata and fills database'
def handle(self, *args, **options): def handle(self, *args, **options):
set_verbosity_level(logger, options.get("verbosity", 1))
if options['all']: if options['all']:
packages = None packages = None
@ -33,4 +34,4 @@ class Command(BaseCommand):
else: else:
packages = [pkg[:-1] for pkg in sys.stdin.readlines()] packages = [pkg[:-1] for pkg in sys.stdin.readlines()]
scan_metadata(packages=packages, quiet=options["quiet"]) scan_metadata(packages=packages, logger=logger)

View File

@ -1,9 +1,14 @@
import sys import sys
import logging
from optparse import make_option from optparse import make_option
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from djeuscan.processing import set_verbosity_level
from djeuscan.processing.scan_portage import scan_portage from djeuscan.processing.scan_portage import scan_portage
logger = logging.getLogger(__name__)
class Command(BaseCommand): class Command(BaseCommand):
_overlays = {} _overlays = {}
@ -35,16 +40,13 @@ class Command(BaseCommand):
default=False, default=False,
help=('Prefetch all versions and packages from DB to ' help=('Prefetch all versions and packages from DB to '
'speedup full scan process.')), 'speedup full scan process.')),
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
) )
args = '[package package ...]' args = '[package package ...]'
help = 'Scans portage tree and fills database' help = 'Scans portage tree and fills database'
def handle(self, *args, **options): def handle(self, *args, **options):
set_verbosity_level(logger, options.get("verbosity", 1))
if options['all']: if options['all']:
packages = None packages = None
@ -59,6 +61,5 @@ class Command(BaseCommand):
purge_packages=options["purge-packages"], purge_packages=options["purge-packages"],
purge_versions=options["purge-versions"], purge_versions=options["purge-versions"],
prefetch=options["prefetch"], prefetch=options["prefetch"],
quiet=options["quiet"], logger=logger,
stdout=self.stdout,
) )

View File

@ -1,10 +1,14 @@
import logging
import sys import sys
from optparse import make_option from optparse import make_option
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from djeuscan.processing import set_verbosity_level
from djeuscan.processing.scan_upstream import scan_upstream from djeuscan.processing.scan_upstream import scan_upstream
logger = logging.getLogger(__name__)
class Command(BaseCommand): class Command(BaseCommand):
_overlays = {} _overlays = {}
@ -20,16 +24,13 @@ class Command(BaseCommand):
dest='purge-versions', dest='purge-versions',
default=False, default=False,
help='Purge old versions'), help='Purge old versions'),
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
) )
args = '<package package ...>' args = '<package package ...>'
help = 'Scans metadata and fills database' help = 'Scans metadata and fills database'
def handle(self, *args, **options): def handle(self, *args, **options):
set_verbosity_level(logger, options.get("verbosity", 1))
if options['all']: if options['all']:
packages = None packages = None
@ -41,6 +42,5 @@ class Command(BaseCommand):
scan_upstream( scan_upstream(
packages=packages, packages=packages,
purge_versions=options["purge-versions"], purge_versions=options["purge-versions"],
quiet=options["quiet"], logger=logger,
stdout=self.stdout,
) )

View File

@ -1,20 +1,19 @@
import logging
from optparse import make_option from optparse import make_option
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from djeuscan.processing import set_verbosity_level
from djeuscan.processing.update_counters import update_counters from djeuscan.processing.update_counters import update_counters
logger = logging.getLogger(__name__)
class Command(BaseCommand): class Command(BaseCommand):
_overlays = {} _overlays = {}
help = 'Update counters' help = 'Update counters'
option_list = BaseCommand.option_list + ( option_list = BaseCommand.option_list + (
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
make_option('--fast', make_option('--fast',
action='store_true', action='store_true',
dest='fast', dest='fast',
@ -28,9 +27,9 @@ class Command(BaseCommand):
) )
def handle(self, *args, **options): def handle(self, *args, **options):
set_verbosity_level(logger, options.get("verbosity", 1))
update_counters( update_counters(
stdout=self.stdout,
fast=options["fast"], fast=options["fast"],
quiet=options["quiet"],
nolog=options["nolog"], nolog=options["nolog"],
logger=logger,
) )

View File

@ -1,3 +1,31 @@
import logging
class FakeLogger(object): class FakeLogger(object):
def __getattr__(self, key): def __getattr__(self, key):
return lambda *x, **y: None return lambda *x, **y: None
def set_verbosity_level(logger, verbosity):
try:
verbosity = int(verbosity)
except (ValueError, TypeError):
return logger
levels = {
0: logging.DEBUG,
1: logging.INFO,
2: logging.WARNING,
3: logging.ERROR,
4: logging.CRITICAL
}
if verbosity < 0:
verbosity = 0
if verbosity > 4:
verbosity = 4
logger.setLevel(levels[verbosity])
return logger

View File

@ -1,4 +1,3 @@
import sys
from gentoolkit.query import Query from gentoolkit.query import Query
from gentoolkit.errors import GentoolkitFatalError from gentoolkit.errors import GentoolkitFatalError
@ -7,12 +6,13 @@ from django.core.management.color import color_style
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from djeuscan.models import Package, Herd, Maintainer from djeuscan.models import Package, Herd, Maintainer
from djeuscan.processing import FakeLogger
class ScanMetadata(object): class ScanMetadata(object):
def __init__(self, quiet=False): def __init__(self, logger=None):
self.quiet = quiet
self.style = color_style() self.style = color_style()
self.logger = logger or FakeLogger()
@commit_on_success @commit_on_success
def scan(self, query=None, obj=None): def scan(self, query=None, obj=None):
@ -22,8 +22,8 @@ class ScanMetadata(object):
) )
if not matches: if not matches:
sys.stderr.write( self.logger.error(
self.style.ERROR("Unknown package '%s'\n" % query) self.style.ERROR("Unknown package '%s'" % query)
) )
return return
@ -43,14 +43,14 @@ class ScanMetadata(object):
obj.homepage = pkg.environment("HOMEPAGE") obj.homepage = pkg.environment("HOMEPAGE")
obj.description = pkg.environment("DESCRIPTION") obj.description = pkg.environment("DESCRIPTION")
except GentoolkitFatalError, err: except GentoolkitFatalError, err:
sys.stderr.write( self.logger.error(
self.style.ERROR( self.style.ERROR(
"Gentoolkit fatal error: '%s'\n" % str(err) "Gentoolkit fatal error: '%s'" % str(err)
) )
) )
if created and not self.quiet: if created:
sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name)) self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name))
if pkg.metadata: if pkg.metadata:
herds = dict( herds = dict(
@ -92,8 +92,8 @@ class ScanMetadata(object):
) )
obj.maintainers.add(maintainer) obj.maintainers.add(maintainer)
except ValidationError: except ValidationError:
sys.stderr.write( self.logger.error(
self.style.ERROR("Bad maintainer: '%s' '%s'\n" % \ self.style.ERROR("Bad maintainer: '%s' '%s'" % \
(maintainer.name, maintainer.email)) (maintainer.name, maintainer.email))
) )
@ -109,8 +109,8 @@ class ScanMetadata(object):
defaults={"email": email} defaults={"email": email}
) )
if created and not self.quiet: if created:
sys.stdout.write('+ [h] %s <%s>\n' % (name, email)) self.logger.info('+ [h] %s <%s>' % (name, email))
herd.email = email herd.email = email
herd.save() herd.save()
@ -129,15 +129,14 @@ class ScanMetadata(object):
) )
if created: if created:
if not self.quiet: self.logger.info(
sys.stdout.write( '+ [m] %s <%s>' % (name.encode('utf-8'), email)
'+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
) )
return maintainer return maintainer
def scan_metadata(packages=None, quiet=False, logger=None): def scan_metadata(packages=None, logger=None):
scan_handler = ScanMetadata(quiet=quiet) scan_handler = ScanMetadata(logger=logger)
if packages is None: if packages is None:
packages = Package.objects.all() packages = Package.objects.all()

View File

@ -1,24 +1,23 @@
import subprocess import subprocess
import portage import portage
import sys
import os import os
import re import re
from django.db.transaction import commit_on_success from django.db.transaction import commit_on_success
from django.core.management.color import color_style from django.core.management.color import color_style
from djeuscan.processing import FakeLogger
from djeuscan.models import Package, Version, VersionLog from djeuscan.models import Package, Version, VersionLog
class ScanPortage(object): class ScanPortage(object):
def __init__(self, stdout=None, no_log=False, purge_packages=False, def __init__(self, logger=None, no_log=False, purge_packages=False,
purge_versions=False, kill_versions=False, quiet=False): purge_versions=False, kill_versions=False):
self.stdout = sys.stdout if stdout is None else stdout self.logger = logger or FakeLogger()
self.no_log = no_log self.no_log = no_log
self.purge_packages = purge_packages self.purge_packages = purge_packages
self.purge_versions = purge_versions self.purge_versions = purge_versions
self.kill_versions = kill_versions self.kill_versions = kill_versions
self.quiet = quiet
self.style = color_style() self.style = color_style()
self._cache = {'packages': {}, 'versions': {}} self._cache = {'packages': {}, 'versions': {}}
@ -93,12 +92,9 @@ class ScanPortage(object):
cmd.extend(['--exact', query]) cmd.extend(['--exact', query])
if self.kill_versions: if self.kill_versions:
if not self.quiet: self.logger.info('Killing existing versions...')
self.stdout.write('Killing existing versions...')
self.stdout.flush()
Version.objects.filter(packaged=True).update(alive=False) Version.objects.filter(packaged=True).update(alive=False)
if not self.quiet: self.logger.info('done')
self.stdout.write('done\n')
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\ output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0] communicate()[0]
@ -108,17 +104,16 @@ class ScanPortage(object):
if not query: if not query:
return return
if self.purge_packages: if self.purge_packages:
if not self.quiet: self.logger.info('- [p] %s' % (query))
sys.stdout.write('- [p] %s\n' % (query))
if '/' in query: if '/' in query:
cat, pkg = portage.catsplit(query) cat, pkg = portage.catsplit(query)
Package.objects.filter(category=cat, name=pkg).delete() Package.objects.filter(category=cat, name=pkg).delete()
else: else:
Package.objects.filter(name=query).delete() Package.objects.filter(name=query).delete()
else: else:
sys.stderr.write( self.logger.error(
self.style.ERROR( self.style.ERROR(
"Unknown package '%s'\n" % query "Unknown package '%s'" % query
) )
) )
return return
@ -156,8 +151,7 @@ class ScanPortage(object):
for package in Package.objects.all(): for package in Package.objects.all():
cp = "%s/%s" % (package.category, package.name) cp = "%s/%s" % (package.category, package.name)
if cp not in packages: if cp not in packages:
if not self.quiet: self.logger.info('- [p] %s' % (package))
sys.stdout.write('- [p] %s\n' % (package))
package.delete() package.delete()
def store_package(self, cat, pkg): def store_package(self, cat, pkg):
@ -172,8 +166,7 @@ class ScanPortage(object):
self.cache_store_package(obj) self.cache_store_package(obj)
if created: if created:
if not self.quiet: self.logger.info('+ [p] %s/%s' % (cat, pkg))
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
# Set all versions dead, then set found versions alive and # Set all versions dead, then set found versions alive and
# delete old versions # delete old versions
@ -219,8 +212,7 @@ class ScanPortage(object):
if not created: if not created:
return return
if not self.quiet: self.logger.info('+ [v] %s' % (obj))
sys.stdout.write('+ [v] %s \n' % (obj))
if overlay == 'gentoo': if overlay == 'gentoo':
package.n_packaged += 1 package.n_packaged += 1
@ -243,7 +235,9 @@ class ScanPortage(object):
@commit_on_success @commit_on_success
def purge_versions(quiet=False, nolog=False): def purge_versions(logger=None, nolog=False):
logger = logger or FakeLogger()
# For each dead versions # For each dead versions
for version in Version.objects.filter(packaged=True, alive=False): for version in Version.objects.filter(packaged=True, alive=False):
if version.overlay == 'gentoo': if version.overlay == 'gentoo':
@ -253,8 +247,7 @@ def purge_versions(quiet=False, nolog=False):
version.package.n_versions -= 1 version.package.n_versions -= 1
version.package.save() version.package.save()
if not quiet: logger.info('- [v] %s' % (version))
sys.stdout.write('- [v] %s\n' % (version))
if nolog: if nolog:
continue continue
@ -272,37 +265,31 @@ def purge_versions(quiet=False, nolog=False):
def scan_portage(packages=None, no_log=False, purge_packages=False, def scan_portage(packages=None, no_log=False, purge_packages=False,
purge_versions=False, prefetch=False, logger=None, purge_versions=False, prefetch=False, logger=None):
quiet=False, stdout=None):
stdout = sys.stdout if stdout is None else stdout
logger = logger or FakeLogger()
kill_versions = False kill_versions = False
if packages is None: if packages is None:
prefetch = True prefetch = True
kill_versions = True kill_versions = True
scan_handler = ScanPortage( scan_handler = ScanPortage(
stdout=stdout, logger=logger,
no_log=no_log, no_log=no_log,
purge_packages=purge_packages, purge_packages=purge_packages,
purge_versions=purge_versions, purge_versions=purge_versions,
kill_versions=kill_versions, kill_versions=kill_versions,
quiet=quiet,
) )
if not quiet: logger.info('Scanning portage tree...')
stdout.write('Scanning portage tree...\n')
if prefetch: if prefetch:
if quiet: logger.info('Prefetching objects...')
stdout.write('Prefetching objects...')
stdout.flush()
for package in Package.objects.all(): for package in Package.objects.all():
scan_handler.cache_store_package(package) scan_handler.cache_store_package(package)
for version in Version.objects.select_related('package').all(): for version in Version.objects.select_related('package').all():
scan_handler.cache_store_version(version) scan_handler.cache_store_version(version)
if quiet: logger.info('done')
stdout.write('done\n')
if packages is None: if packages is None:
scan_handler.scan() scan_handler.scan()
@ -314,7 +301,6 @@ def scan_portage(packages=None, no_log=False, purge_packages=False,
scan_handler.scan(pkg) scan_handler.scan(pkg)
if purge_versions: if purge_versions:
purge_versions(quiet, no_log) purge_versions(logger=logger, no_log=no_log)
if not quiet: logger.info('Done.')
stdout.write('Done.\n')

View File

@ -1,5 +1,4 @@
import portage import portage
import sys
from django.utils import timezone from django.utils import timezone
from django.db.transaction import commit_on_success from django.db.transaction import commit_on_success
@ -7,12 +6,13 @@ from django.db.transaction import commit_on_success
from euscan import CONFIG, output from euscan import CONFIG, output
from euscan.scan import scan_upstream as euscan_scan_upstream from euscan.scan import scan_upstream as euscan_scan_upstream
from djeuscan.processing import FakeLogger
from djeuscan.models import Package, Version, EuscanResult, VersionLog from djeuscan.models import Package, Version, EuscanResult, VersionLog
class ScanUpstream(object): class ScanUpstream(object):
def __init__(self, quiet=False): def __init__(self, logger=None):
self.quiet = quiet self.logger = logger or FakeLogger()
def scan(self, package): def scan(self, package):
CONFIG["format"] = "dict" CONFIG["format"] = "dict"
@ -52,8 +52,8 @@ class ScanUpstream(object):
obj, created = Package.objects.get_or_create(category=cat, name=pkg) obj, created = Package.objects.get_or_create(category=cat, name=pkg)
if created and not self.quiet: if created:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg)) self.logger.info('+ [p] %s/%s' % (cat, pkg))
# Set all versions dead, then set found versions alive and # Set all versions dead, then set found versions alive and
# delete old versions # delete old versions
@ -76,8 +76,7 @@ class ScanUpstream(object):
if not created: if not created:
return return
if not self.quiet: self.logger.info('+ [u] %s %s' % (obj, url))
sys.stdout.write('+ [u] %s %s\n' % (obj, url))
VersionLog.objects.create( VersionLog.objects.create(
package=package, package=package,
@ -93,7 +92,9 @@ class ScanUpstream(object):
@commit_on_success @commit_on_success
def purge_versions(quiet=False): def purge_versions(logger=None):
logger = logger or FakeLogger()
# For each dead versions # For each dead versions
for version in Version.objects.filter(packaged=False, alive=False): for version in Version.objects.filter(packaged=False, alive=False):
VersionLog.objects.create( VersionLog.objects.create(
@ -108,20 +109,17 @@ def purge_versions(quiet=False):
version.package.n_versions -= 1 version.package.n_versions -= 1
version.package.save() version.package.save()
if not quiet: logger.info('- [u] %s %s' % (version, version.urls))
sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
Version.objects.filter(packaged=False, alive=False).delete() Version.objects.filter(packaged=False, alive=False).delete()
def scan_upstream(packages=None, purge_versions=False, quiet=False, def scan_upstream(packages=None, purge_versions=False,
logger=None, stdout=None): logger=None):
logger = logger or FakeLogger()
stdout = sys.stdout if stdout is None else stdout scan_handler = ScanUpstream(logger=logger)
scan_handler = ScanUpstream(quiet) logger.info('Scanning upstream...')
if not quiet:
stdout.write('Scanning upstream...\n')
if packages is None: if packages is None:
packages = Package.objects.all() packages = Package.objects.all()
@ -133,7 +131,6 @@ def scan_upstream(packages=None, purge_versions=False, quiet=False,
scan_handler.scan(pkg) scan_handler.scan(pkg)
if purge_versions: if purge_versions:
purge_versions(quiet) purge_versions(logger=logger)
if not quiet: logger.info('Done.')
stdout.write('Done.\n')

View File

@ -1,5 +1,3 @@
import sys
from django.db.transaction import commit_on_success from django.db.transaction import commit_on_success
from django.utils import timezone from django.utils import timezone
@ -39,13 +37,8 @@ def _add_last_ver(storage, version):
@commit_on_success @commit_on_success
def update_counters(stdout=None, fast=False, quiet=False, nolog=False, def update_counters(fast=False, nolog=False, logger=None):
logger=None): logger = logger or FakeLogger()
if logger is None:
logger = FakeLogger() # TODO: write log messages
if stdout is None:
stdout = sys.stdout
now = timezone.now() now = timezone.now()
@ -164,20 +157,17 @@ def update_counters(stdout=None, fast=False, quiet=False, nolog=False,
return return
for clog in categories.values(): for clog in categories.values():
if not quiet: logger.info('+ [cl] %s\n' % clog)
stdout.write('+ [cl] %s\n' % clog)
charts.rrd_update('category-%s' % clog.category, now, clog) charts.rrd_update('category-%s' % clog.category, now, clog)
clog.save() clog.save()
for hlog in herds.values(): for hlog in herds.values():
if not quiet: logger.info('+ [hl] %s\n' % hlog)
stdout.write('+ [hl] %s\n' % hlog)
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog) charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
hlog.save() hlog.save()
for mlog in maintainers.values(): for mlog in maintainers.values():
if not quiet: logger.info('+ [ml] %s\n' % mlog)
stdout.write('+ [ml] %s\n' % mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog) charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save() mlog.save()

View File

@ -181,12 +181,22 @@ INSTALLED_APPS = (
LOGGING = { LOGGING = {
'version': 1, 'version': 1,
'disable_existing_loggers': False, 'disable_existing_loggers': False,
'formatters': {
'simple': {
'format': '%(levelname)s %(asctime)s %(message)s'
},
},
'filters': { 'filters': {
'require_debug_false': { 'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse' '()': 'django.utils.log.RequireDebugFalse'
} }
}, },
'handlers': { 'handlers': {
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'simple'
},
'mail_admins': { 'mail_admins': {
'level': 'ERROR', 'level': 'ERROR',
'filters': ['require_debug_false'], 'filters': ['require_debug_false'],
@ -199,6 +209,11 @@ LOGGING = {
'level': 'ERROR', 'level': 'ERROR',
'propagate': True, 'propagate': True,
}, },
'djeuscan': {
'handlers': ['console'],
'level': 'INFO',
'propagate': True
}
} }
} }