src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

euscanwww: use logging in djeuscan.processing

Browse Source 
Signed-off-by: volpino <fox91@anche.no>
This commit is contained in:
volpino
2012-06-22 11:31:15 +02:00
parent 9af5548178
commit 779ccdf5d3
11 changed files with 142 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
euscanwww/djeuscan/processing/__init__.py
View File

@@ -1,3 +1,31 @@
import logging
class FakeLogger(object):
def __getattr__(self, key):
return lambda *x, **y: None
def set_verbosity_level(logger, verbosity):
try:
verbosity = int(verbosity)
except (ValueError, TypeError):
return logger
levels = {
0: logging.DEBUG,
1: logging.INFO,
2: logging.WARNING,
3: logging.ERROR,
4: logging.CRITICAL
}
if verbosity < 0:
verbosity = 0
if verbosity > 4:
verbosity = 4
logger.setLevel(levels[verbosity])
return logger

37
euscanwww/djeuscan/processing/scan_metadata.py
View File

@@ -1,4 +1,3 @@
import sys
from gentoolkit.query import Query
from gentoolkit.errors import GentoolkitFatalError
@@ -7,12 +6,13 @@ from django.core.management.color import color_style
from django.core.exceptions import ValidationError
from djeuscan.models import Package, Herd, Maintainer
from djeuscan.processing import FakeLogger
class ScanMetadata(object):
def __init__(self, quiet=False):
self.quiet = quiet
def __init__(self, logger=None):
self.style = color_style()
self.logger = logger or FakeLogger()
@commit_on_success
def scan(self, query=None, obj=None):
@@ -22,8 +22,8 @@ class ScanMetadata(object):
)
if not matches:
sys.stderr.write(
self.style.ERROR("Unknown package '%s'\n" % query)
self.logger.error(
self.style.ERROR("Unknown package '%s'" % query)
)
return
@@ -43,14 +43,14 @@ class ScanMetadata(object):
obj.homepage = pkg.environment("HOMEPAGE")
obj.description = pkg.environment("DESCRIPTION")
except GentoolkitFatalError, err:
sys.stderr.write(
self.logger.error(
self.style.ERROR(
"Gentoolkit fatal error: '%s'\n" % str(err)
"Gentoolkit fatal error: '%s'" % str(err)
)
)
if created and not self.quiet:
sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
if created:
self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name))
if pkg.metadata:
herds = dict(
@@ -92,8 +92,8 @@ class ScanMetadata(object):
)
obj.maintainers.add(maintainer)
except ValidationError:
sys.stderr.write(
self.style.ERROR("Bad maintainer: '%s' '%s'\n" % \
self.logger.error(
self.style.ERROR("Bad maintainer: '%s' '%s'" % \
(maintainer.name, maintainer.email))
)
@@ -109,8 +109,8 @@ class ScanMetadata(object):
defaults={"email": email}
)
if created and not self.quiet:
sys.stdout.write('+ [h] %s <%s>\n' % (name, email))
if created:
self.logger.info('+ [h] %s <%s>' % (name, email))
herd.email = email
herd.save()
@@ -129,15 +129,14 @@ class ScanMetadata(object):
)
if created:
if not self.quiet:
sys.stdout.write(
'+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
)
self.logger.info(
'+ [m] %s <%s>' % (name.encode('utf-8'), email)
)
return maintainer
def scan_metadata(packages=None, quiet=False, logger=None):
scan_handler = ScanMetadata(quiet=quiet)
def scan_metadata(packages=None, logger=None):
scan_handler = ScanMetadata(logger=logger)
if packages is None:
packages = Package.objects.all()

62
euscanwww/djeuscan/processing/scan_portage.py
View File

@@ -1,24 +1,23 @@
import subprocess
import portage
import sys
import os
import re
from django.db.transaction import commit_on_success
from django.core.management.color import color_style
from djeuscan.processing import FakeLogger
from djeuscan.models import Package, Version, VersionLog
class ScanPortage(object):
def __init__(self, stdout=None, no_log=False, purge_packages=False,
purge_versions=False, kill_versions=False, quiet=False):
self.stdout = sys.stdout if stdout is None else stdout
def __init__(self, logger=None, no_log=False, purge_packages=False,
purge_versions=False, kill_versions=False):
self.logger = logger or FakeLogger()
self.no_log = no_log
self.purge_packages = purge_packages
self.purge_versions = purge_versions
self.kill_versions = kill_versions
self.quiet = quiet
self.style = color_style()
self._cache = {'packages': {}, 'versions': {}}
@@ -93,12 +92,9 @@ class ScanPortage(object):
cmd.extend(['--exact', query])
if self.kill_versions:
if not self.quiet:
self.stdout.write('Killing existing versions...')
self.stdout.flush()
self.logger.info('Killing existing versions...')
Version.objects.filter(packaged=True).update(alive=False)
if not self.quiet:
self.stdout.write('done\n')
self.logger.info('done')
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
@@ -108,17 +104,16 @@ class ScanPortage(object):
if not query:
return
if self.purge_packages:
if not self.quiet:
sys.stdout.write('- [p] %s\n' % (query))
self.logger.info('- [p] %s' % (query))
if '/' in query:
cat, pkg = portage.catsplit(query)
Package.objects.filter(category=cat, name=pkg).delete()
else:
Package.objects.filter(name=query).delete()
else:
sys.stderr.write(
self.logger.error(
self.style.ERROR(
"Unknown package '%s'\n" % query
"Unknown package '%s'" % query
)
)
return
@@ -156,8 +151,7 @@ class ScanPortage(object):
for package in Package.objects.all():
cp = "%s/%s" % (package.category, package.name)
if cp not in packages:
if not self.quiet:
sys.stdout.write('- [p] %s\n' % (package))
self.logger.info('- [p] %s' % (package))
package.delete()
def store_package(self, cat, pkg):
@@ -172,8 +166,7 @@ class ScanPortage(object):
self.cache_store_package(obj)
if created:
if not self.quiet:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
self.logger.info('+ [p] %s/%s' % (cat, pkg))
# Set all versions dead, then set found versions alive and
# delete old versions
@@ -219,8 +212,7 @@ class ScanPortage(object):
if not created:
return
if not self.quiet:
sys.stdout.write('+ [v] %s \n' % (obj))
self.logger.info('+ [v] %s' % (obj))
if overlay == 'gentoo':
package.n_packaged += 1
@@ -243,7 +235,9 @@ class ScanPortage(object):
@commit_on_success
def purge_versions(quiet=False, nolog=False):
def purge_versions(logger=None, nolog=False):
logger = logger or FakeLogger()
# For each dead versions
for version in Version.objects.filter(packaged=True, alive=False):
if version.overlay == 'gentoo':
@@ -253,8 +247,7 @@ def purge_versions(quiet=False, nolog=False):
version.package.n_versions -= 1
version.package.save()
if not quiet:
sys.stdout.write('- [v] %s\n' % (version))
logger.info('- [v] %s' % (version))
if nolog:
continue
@@ -272,37 +265,31 @@ def purge_versions(quiet=False, nolog=False):
def scan_portage(packages=None, no_log=False, purge_packages=False,
purge_versions=False, prefetch=False, logger=None,
quiet=False, stdout=None):
stdout = sys.stdout if stdout is None else stdout
purge_versions=False, prefetch=False, logger=None):
logger = logger or FakeLogger()
kill_versions = False
if packages is None:
prefetch = True
kill_versions = True
scan_handler = ScanPortage(
stdout=stdout,
logger=logger,
no_log=no_log,
purge_packages=purge_packages,
purge_versions=purge_versions,
kill_versions=kill_versions,
quiet=quiet,
)
if not quiet:
stdout.write('Scanning portage tree...\n')
logger.info('Scanning portage tree...')
if prefetch:
if quiet:
stdout.write('Prefetching objects...')
stdout.flush()
logger.info('Prefetching objects...')
for package in Package.objects.all():
scan_handler.cache_store_package(package)
for version in Version.objects.select_related('package').all():
scan_handler.cache_store_version(version)
if quiet:
stdout.write('done\n')
logger.info('done')
if packages is None:
scan_handler.scan()
@@ -314,7 +301,6 @@ def scan_portage(packages=None, no_log=False, purge_packages=False,
scan_handler.scan(pkg)
if purge_versions:
purge_versions(quiet, no_log)
purge_versions(logger=logger, no_log=no_log)
if not quiet:
stdout.write('Done.\n')
logger.info('Done.')

37
euscanwww/djeuscan/processing/scan_upstream.py
View File

@@ -1,5 +1,4 @@
import portage
import sys
from django.utils import timezone
from django.db.transaction import commit_on_success
@@ -7,12 +6,13 @@ from django.db.transaction import commit_on_success
from euscan import CONFIG, output
from euscan.scan import scan_upstream as euscan_scan_upstream
from djeuscan.processing import FakeLogger
from djeuscan.models import Package, Version, EuscanResult, VersionLog
class ScanUpstream(object):
def __init__(self, quiet=False):
self.quiet = quiet
def __init__(self, logger=None):
self.logger = logger or FakeLogger()
def scan(self, package):
CONFIG["format"] = "dict"
@@ -52,8 +52,8 @@ class ScanUpstream(object):
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
if created and not self.quiet:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
if created:
self.logger.info('+ [p] %s/%s' % (cat, pkg))
# Set all versions dead, then set found versions alive and
# delete old versions
@@ -76,8 +76,7 @@ class ScanUpstream(object):
if not created:
return
if not self.quiet:
sys.stdout.write('+ [u] %s %s\n' % (obj, url))
self.logger.info('+ [u] %s %s' % (obj, url))
VersionLog.objects.create(
package=package,
@@ -93,7 +92,9 @@ class ScanUpstream(object):
@commit_on_success
def purge_versions(quiet=False):
def purge_versions(logger=None):
logger = logger or FakeLogger()
# For each dead versions
for version in Version.objects.filter(packaged=False, alive=False):
VersionLog.objects.create(
@@ -108,20 +109,17 @@ def purge_versions(quiet=False):
version.package.n_versions -= 1
version.package.save()
if not quiet:
sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
logger.info('- [u] %s %s' % (version, version.urls))
Version.objects.filter(packaged=False, alive=False).delete()
def scan_upstream(packages=None, purge_versions=False, quiet=False,
logger=None, stdout=None):
def scan_upstream(packages=None, purge_versions=False,
logger=None):
logger = logger or FakeLogger()
stdout = sys.stdout if stdout is None else stdout
scan_handler = ScanUpstream(logger=logger)
scan_handler = ScanUpstream(quiet)
if not quiet:
stdout.write('Scanning upstream...\n')
logger.info('Scanning upstream...')
if packages is None:
packages = Package.objects.all()
@@ -133,7 +131,6 @@ def scan_upstream(packages=None, purge_versions=False, quiet=False,
scan_handler.scan(pkg)
if purge_versions:
purge_versions(quiet)
purge_versions(logger=logger)
if not quiet:
stdout.write('Done.\n')
logger.info('Done.')

20
euscanwww/djeuscan/processing/update_counters.py
View File

@@ -1,5 +1,3 @@
import sys
from django.db.transaction import commit_on_success
from django.utils import timezone
@@ -39,13 +37,8 @@ def _add_last_ver(storage, version):
@commit_on_success
def update_counters(stdout=None, fast=False, quiet=False, nolog=False,
logger=None):
if logger is None:
logger = FakeLogger() # TODO: write log messages
if stdout is None:
stdout = sys.stdout
def update_counters(fast=False, nolog=False, logger=None):
logger = logger or FakeLogger()
now = timezone.now()
@@ -164,20 +157,17 @@ def update_counters(stdout=None, fast=False, quiet=False, nolog=False,
return
for clog in categories.values():
if not quiet:
stdout.write('+ [cl] %s\n' % clog)
logger.info('+ [cl] %s\n' % clog)
charts.rrd_update('category-%s' % clog.category, now, clog)
clog.save()
for hlog in herds.values():
if not quiet:
stdout.write('+ [hl] %s\n' % hlog)
logger.info('+ [hl] %s\n' % hlog)
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
hlog.save()
for mlog in maintainers.values():
if not quiet:
stdout.write('+ [ml] %s\n' % mlog)
logger.info('+ [ml] %s\n' % mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save()