src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

euscanwww: use logging in djeuscan.processing

Browse Source 
Signed-off-by: volpino <fox91@anche.no>
This commit is contained in:
volpino 2012-06-22 11:31:15 +02:00
parent 9af5548178
commit 779ccdf5d3
11 changed files with 142 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
euscanwww/djeuscan/management/commands/regen_rrds.py
View File

@ -1,10 +1,16 @@
import logging
from django.core.management.base import BaseCommand
from djeuscan.processing import set_verbosity_level
from djeuscan.processing.regen_rrds import regen_rrds
logger = logging.getLogger(__name__)
class Command(BaseCommand):
_overlays = {}
help = 'Regenerate rrd database'
def handle(self, *args, **options):
regen_rrds()
set_verbosity_level(logger, options.get("verbosity", 1))
regen_rrds(logger=logger)

13
euscanwww/djeuscan/management/commands/scan_metadata.py
View File

@ -1,10 +1,14 @@
import sys
import logging
from optparse import make_option
from django.core.management.base import BaseCommand
from djeuscan.processing import set_verbosity_level
from djeuscan.processing.scan_metadata import scan_metadata
logger = logging.getLogger(__name__)
class Command(BaseCommand):
_overlays = {}
@ -15,16 +19,13 @@ class Command(BaseCommand):
dest='all',
default=False,
help='Scan all packages'),
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
)
args = '<package package ...>'
help = 'Scans metadata and fills database'
def handle(self, *args, **options):
set_verbosity_level(logger, options.get("verbosity", 1))
if options['all']:
packages = None
@ -33,4 +34,4 @@ class Command(BaseCommand):
else:
packages = [pkg[:-1] for pkg in sys.stdin.readlines()]
scan_metadata(packages=packages, quiet=options["quiet"])
scan_metadata(packages=packages, logger=logger)

15
euscanwww/djeuscan/management/commands/scan_portage.py
View File

@ -1,9 +1,14 @@
import sys
import logging
from optparse import make_option
from django.core.management.base import BaseCommand
from djeuscan.processing import set_verbosity_level
from djeuscan.processing.scan_portage import scan_portage
logger = logging.getLogger(__name__)
class Command(BaseCommand):
_overlays = {}
@ -35,16 +40,13 @@ class Command(BaseCommand):
default=False,
help=('Prefetch all versions and packages from DB to '
'speedup full scan process.')),
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
)
args = '[package package ...]'
help = 'Scans portage tree and fills database'
def handle(self, *args, **options):
set_verbosity_level(logger, options.get("verbosity", 1))
if options['all']:
packages = None
@ -59,6 +61,5 @@ class Command(BaseCommand):
purge_packages=options["purge-packages"],
purge_versions=options["purge-versions"],
prefetch=options["prefetch"],
quiet=options["quiet"],
stdout=self.stdout,
logger=logger,
)

14
euscanwww/djeuscan/management/commands/scan_upstream.py
View File

@ -1,10 +1,14 @@
import logging
import sys
from optparse import make_option
from django.core.management.base import BaseCommand
from djeuscan.processing import set_verbosity_level
from djeuscan.processing.scan_upstream import scan_upstream
logger = logging.getLogger(__name__)
class Command(BaseCommand):
_overlays = {}
@ -20,16 +24,13 @@ class Command(BaseCommand):
dest='purge-versions',
default=False,
help='Purge old versions'),
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
)
args = '<package package ...>'
help = 'Scans metadata and fills database'
def handle(self, *args, **options):
set_verbosity_level(logger, options.get("verbosity", 1))
if options['all']:
packages = None
@ -41,6 +42,5 @@ class Command(BaseCommand):
scan_upstream(
packages=packages,
purge_versions=options["purge-versions"],
quiet=options["quiet"],
stdout=self.stdout,
logger=logger,
)

13
euscanwww/djeuscan/management/commands/update_counters.py
View File

@ -1,20 +1,19 @@
import logging
from optparse import make_option
from django.core.management.base import BaseCommand
from djeuscan.processing import set_verbosity_level
from djeuscan.processing.update_counters import update_counters
logger = logging.getLogger(__name__)
class Command(BaseCommand):
_overlays = {}
help = 'Update counters'
option_list = BaseCommand.option_list + (
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
make_option('--fast',
action='store_true',
dest='fast',
@ -28,9 +27,9 @@ class Command(BaseCommand):
)
def handle(self, *args, **options):
set_verbosity_level(logger, options.get("verbosity", 1))
update_counters(
stdout=self.stdout,
fast=options["fast"],
quiet=options["quiet"],
nolog=options["nolog"],
logger=logger,
)

28
euscanwww/djeuscan/processing/__init__.py
View File

@ -1,3 +1,31 @@
import logging
class FakeLogger(object):
def __getattr__(self, key):
return lambda *x, **y: None
def set_verbosity_level(logger, verbosity):
try:
verbosity = int(verbosity)
except (ValueError, TypeError):
return logger
levels = {
0: logging.DEBUG,
1: logging.INFO,
2: logging.WARNING,
3: logging.ERROR,
4: logging.CRITICAL
}
if verbosity < 0:
verbosity = 0
if verbosity > 4:
verbosity = 4
logger.setLevel(levels[verbosity])
return logger

35
euscanwww/djeuscan/processing/scan_metadata.py
View File

@ -1,4 +1,3 @@
import sys
from gentoolkit.query import Query
from gentoolkit.errors import GentoolkitFatalError
@ -7,12 +6,13 @@ from django.core.management.color import color_style
from django.core.exceptions import ValidationError
from djeuscan.models import Package, Herd, Maintainer
from djeuscan.processing import FakeLogger
class ScanMetadata(object):
def __init__(self, quiet=False):
self.quiet = quiet
def __init__(self, logger=None):
self.style = color_style()
self.logger = logger or FakeLogger()
@commit_on_success
def scan(self, query=None, obj=None):
@ -22,8 +22,8 @@ class ScanMetadata(object):
)
if not matches:
sys.stderr.write(
self.style.ERROR("Unknown package '%s'\n" % query)
self.logger.error(
self.style.ERROR("Unknown package '%s'" % query)
)
return
@ -43,14 +43,14 @@ class ScanMetadata(object):
obj.homepage = pkg.environment("HOMEPAGE")
obj.description = pkg.environment("DESCRIPTION")
except GentoolkitFatalError, err:
sys.stderr.write(
self.logger.error(
self.style.ERROR(
"Gentoolkit fatal error: '%s'\n" % str(err)
"Gentoolkit fatal error: '%s'" % str(err)
)
)
if created and not self.quiet:
sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
if created:
self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name))
if pkg.metadata:
herds = dict(
@ -92,8 +92,8 @@ class ScanMetadata(object):
)
obj.maintainers.add(maintainer)
except ValidationError:
sys.stderr.write(
self.style.ERROR("Bad maintainer: '%s' '%s'\n" % \
self.logger.error(
self.style.ERROR("Bad maintainer: '%s' '%s'" % \
(maintainer.name, maintainer.email))
)
@ -109,8 +109,8 @@ class ScanMetadata(object):
defaults={"email": email}
)
if created and not self.quiet:
sys.stdout.write('+ [h] %s <%s>\n' % (name, email))
if created:
self.logger.info('+ [h] %s <%s>' % (name, email))
herd.email = email
herd.save()
@ -129,15 +129,14 @@ class ScanMetadata(object):
)
if created:
if not self.quiet:
sys.stdout.write(
'+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
self.logger.info(
'+ [m] %s <%s>' % (name.encode('utf-8'), email)
)
return maintainer
def scan_metadata(packages=None, quiet=False, logger=None):
scan_handler = ScanMetadata(quiet=quiet)
def scan_metadata(packages=None, logger=None):
scan_handler = ScanMetadata(logger=logger)
if packages is None:
packages = Package.objects.all()

62
euscanwww/djeuscan/processing/scan_portage.py
View File

@ -1,24 +1,23 @@
import subprocess
import portage
import sys
import os
import re
from django.db.transaction import commit_on_success
from django.core.management.color import color_style
from djeuscan.processing import FakeLogger
from djeuscan.models import Package, Version, VersionLog
class ScanPortage(object):
def __init__(self, stdout=None, no_log=False, purge_packages=False,
purge_versions=False, kill_versions=False, quiet=False):
self.stdout = sys.stdout if stdout is None else stdout
def __init__(self, logger=None, no_log=False, purge_packages=False,
purge_versions=False, kill_versions=False):
self.logger = logger or FakeLogger()
self.no_log = no_log
self.purge_packages = purge_packages
self.purge_versions = purge_versions
self.kill_versions = kill_versions
self.quiet = quiet
self.style = color_style()
self._cache = {'packages': {}, 'versions': {}}
@ -93,12 +92,9 @@ class ScanPortage(object):
cmd.extend(['--exact', query])
if self.kill_versions:
if not self.quiet:
self.stdout.write('Killing existing versions...')
self.stdout.flush()
self.logger.info('Killing existing versions...')
Version.objects.filter(packaged=True).update(alive=False)
if not self.quiet:
self.stdout.write('done\n')
self.logger.info('done')
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
@ -108,17 +104,16 @@ class ScanPortage(object):
if not query:
return
if self.purge_packages:
if not self.quiet:
sys.stdout.write('- [p] %s\n' % (query))
self.logger.info('- [p] %s' % (query))
if '/' in query:
cat, pkg = portage.catsplit(query)
Package.objects.filter(category=cat, name=pkg).delete()
else:
Package.objects.filter(name=query).delete()
else:
sys.stderr.write(
self.logger.error(
self.style.ERROR(
"Unknown package '%s'\n" % query
"Unknown package '%s'" % query
)
)
return
@ -156,8 +151,7 @@ class ScanPortage(object):
for package in Package.objects.all():
cp = "%s/%s" % (package.category, package.name)
if cp not in packages:
if not self.quiet:
sys.stdout.write('- [p] %s\n' % (package))
self.logger.info('- [p] %s' % (package))
package.delete()
def store_package(self, cat, pkg):
@ -172,8 +166,7 @@ class ScanPortage(object):
self.cache_store_package(obj)
if created:
if not self.quiet:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
self.logger.info('+ [p] %s/%s' % (cat, pkg))
# Set all versions dead, then set found versions alive and
# delete old versions
@ -219,8 +212,7 @@ class ScanPortage(object):
if not created:
return
if not self.quiet:
sys.stdout.write('+ [v] %s \n' % (obj))
self.logger.info('+ [v] %s' % (obj))
if overlay == 'gentoo':
package.n_packaged += 1
@ -243,7 +235,9 @@ class ScanPortage(object):
@commit_on_success
def purge_versions(quiet=False, nolog=False):
def purge_versions(logger=None, nolog=False):
logger = logger or FakeLogger()
# For each dead versions
for version in Version.objects.filter(packaged=True, alive=False):
if version.overlay == 'gentoo':
@ -253,8 +247,7 @@ def purge_versions(quiet=False, nolog=False):
version.package.n_versions -= 1
version.package.save()
if not quiet:
sys.stdout.write('- [v] %s\n' % (version))
logger.info('- [v] %s' % (version))
if nolog:
continue
@ -272,37 +265,31 @@ def purge_versions(quiet=False, nolog=False):
def scan_portage(packages=None, no_log=False, purge_packages=False,
purge_versions=False, prefetch=False, logger=None,
quiet=False, stdout=None):
stdout = sys.stdout if stdout is None else stdout
purge_versions=False, prefetch=False, logger=None):
logger = logger or FakeLogger()
kill_versions = False
if packages is None:
prefetch = True
kill_versions = True
scan_handler = ScanPortage(
stdout=stdout,
logger=logger,
no_log=no_log,
purge_packages=purge_packages,
purge_versions=purge_versions,
kill_versions=kill_versions,
quiet=quiet,
)
if not quiet:
stdout.write('Scanning portage tree...\n')
logger.info('Scanning portage tree...')
if prefetch:
if quiet:
stdout.write('Prefetching objects...')
stdout.flush()
logger.info('Prefetching objects...')
for package in Package.objects.all():
scan_handler.cache_store_package(package)
for version in Version.objects.select_related('package').all():
scan_handler.cache_store_version(version)
if quiet:
stdout.write('done\n')
logger.info('done')
if packages is None:
scan_handler.scan()
@ -314,7 +301,6 @@ def scan_portage(packages=None, no_log=False, purge_packages=False,
scan_handler.scan(pkg)
if purge_versions:
purge_versions(quiet, no_log)
purge_versions(logger=logger, no_log=no_log)
if not quiet:
stdout.write('Done.\n')
logger.info('Done.')

37
euscanwww/djeuscan/processing/scan_upstream.py
View File

@ -1,5 +1,4 @@
import portage
import sys
from django.utils import timezone
from django.db.transaction import commit_on_success
@ -7,12 +6,13 @@ from django.db.transaction import commit_on_success
from euscan import CONFIG, output
from euscan.scan import scan_upstream as euscan_scan_upstream
from djeuscan.processing import FakeLogger
from djeuscan.models import Package, Version, EuscanResult, VersionLog
class ScanUpstream(object):
def __init__(self, quiet=False):
self.quiet = quiet
def __init__(self, logger=None):
self.logger = logger or FakeLogger()
def scan(self, package):
CONFIG["format"] = "dict"
@ -52,8 +52,8 @@ class ScanUpstream(object):
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
if created and not self.quiet:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
if created:
self.logger.info('+ [p] %s/%s' % (cat, pkg))
# Set all versions dead, then set found versions alive and
# delete old versions
@ -76,8 +76,7 @@ class ScanUpstream(object):
if not created:
return
if not self.quiet:
sys.stdout.write('+ [u] %s %s\n' % (obj, url))
self.logger.info('+ [u] %s %s' % (obj, url))
VersionLog.objects.create(
package=package,
@ -93,7 +92,9 @@ class ScanUpstream(object):
@commit_on_success
def purge_versions(quiet=False):
def purge_versions(logger=None):
logger = logger or FakeLogger()
# For each dead versions
for version in Version.objects.filter(packaged=False, alive=False):
VersionLog.objects.create(
@ -108,20 +109,17 @@ def purge_versions(quiet=False):
version.package.n_versions -= 1
version.package.save()
if not quiet:
sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
logger.info('- [u] %s %s' % (version, version.urls))
Version.objects.filter(packaged=False, alive=False).delete()
def scan_upstream(packages=None, purge_versions=False, quiet=False,
logger=None, stdout=None):
def scan_upstream(packages=None, purge_versions=False,
logger=None):
logger = logger or FakeLogger()
stdout = sys.stdout if stdout is None else stdout
scan_handler = ScanUpstream(logger=logger)
scan_handler = ScanUpstream(quiet)
if not quiet:
stdout.write('Scanning upstream...\n')
logger.info('Scanning upstream...')
if packages is None:
packages = Package.objects.all()
@ -133,7 +131,6 @@ def scan_upstream(packages=None, purge_versions=False, quiet=False,
scan_handler.scan(pkg)
if purge_versions:
purge_versions(quiet)
purge_versions(logger=logger)
if not quiet:
stdout.write('Done.\n')
logger.info('Done.')

20
euscanwww/djeuscan/processing/update_counters.py
View File

@ -1,5 +1,3 @@
import sys
from django.db.transaction import commit_on_success
from django.utils import timezone
@ -39,13 +37,8 @@ def _add_last_ver(storage, version):
@commit_on_success
def update_counters(stdout=None, fast=False, quiet=False, nolog=False,
logger=None):
if logger is None:
logger = FakeLogger() # TODO: write log messages
if stdout is None:
stdout = sys.stdout
def update_counters(fast=False, nolog=False, logger=None):
logger = logger or FakeLogger()
now = timezone.now()
@ -164,20 +157,17 @@ def update_counters(stdout=None, fast=False, quiet=False, nolog=False,
return
for clog in categories.values():
if not quiet:
stdout.write('+ [cl] %s\n' % clog)
logger.info('+ [cl] %s\n' % clog)
charts.rrd_update('category-%s' % clog.category, now, clog)
clog.save()
for hlog in herds.values():
if not quiet:
stdout.write('+ [hl] %s\n' % hlog)
logger.info('+ [hl] %s\n' % hlog)
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
hlog.save()
for mlog in maintainers.values():
if not quiet:
stdout.write('+ [ml] %s\n' % mlog)
logger.info('+ [ml] %s\n' % mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save()

15
euscanwww/euscanwww/settings.py
View File

@ -181,12 +181,22 @@ INSTALLED_APPS = (
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'simple': {
'format': '%(levelname)s %(asctime)s %(message)s'
},
},
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse'
}
},
'handlers': {
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'simple'
},
'mail_admins': {
'level': 'ERROR',
'filters': ['require_debug_false'],
@ -199,6 +209,11 @@ LOGGING = {
'level': 'ERROR',
'propagate': True,
},
'djeuscan': {
'handlers': ['console'],
'level': 'INFO',
'propagate': True
}
}
}