euscanwww: move processing to scan and misc, enhance update_portage_tree
update_portage_tree() now: - watch stderr and stdout for each command - use layman command instead of layman API for sync because layman API doesn't work when stdout or stderr is not a real file (we could probably work around that with pipes and epoll) - use egencache instead of emerge to generate cache - export PORTAGE_CONFIGROOT, ROOT, EIX_CACHEFILE etc.. so they are used everywhere Signed-off-by: Corentin Chary <corentin.chary@gmail.com>
This commit is contained in:
Corentin Chary
5
euscanwww/djeuscan/processing/misc/__init__.py
Normal file
5
euscanwww/djeuscan/processing/misc/__init__.py
Normal file
@ -0,0 +1,5 @@
|
||||
__all__ = ["regen_rrds", "update_counters", "update_portage_trees"]
|
||||
|
||||
from regen_rrds import regen_rrds
|
||||
from update_counters import update_counters
|
||||
from update_portage_trees import update_portage_trees
|
||||
30
euscanwww/djeuscan/processing/misc/regen_rrds.py
Normal file
30
euscanwww/djeuscan/processing/misc/regen_rrds.py
Normal file
@ -0,0 +1,30 @@
|
||||
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
|
||||
from djeuscan import charts
|
||||
|
||||
from djeuscan.processing import FakeLogger
|
||||
|
||||
def regen_rrds(logger=None):
|
||||
"""
|
||||
Regenerates the rrd database
|
||||
"""
|
||||
|
||||
if logger is None:
|
||||
logger = FakeLogger()
|
||||
|
||||
logger.info("Regenering RRDs for world")
|
||||
for wlog in WorldLog.objects.all():
|
||||
charts.rrd_update('world', wlog.datetime, wlog)
|
||||
|
||||
logger.info("Regenering RRDs for categories")
|
||||
for clog in CategoryLog.objects.all():
|
||||
charts.rrd_update('category-%s' % clog.category,
|
||||
clog.datetime, clog)
|
||||
|
||||
logger.info("Regenering RRDs for herds")
|
||||
for hlog in HerdLog.objects.all():
|
||||
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
|
||||
|
||||
logger.info("Regenering RRDs for maintainers")
|
||||
for mlog in MaintainerLog.objects.all():
|
||||
charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
|
||||
mlog.datetime, mlog)
|
||||
175
euscanwww/djeuscan/processing/misc/update_counters.py
Normal file
175
euscanwww/djeuscan/processing/misc/update_counters.py
Normal file
@ -0,0 +1,175 @@
|
||||
from django.db.transaction import commit_on_success
|
||||
from django.utils import timezone
|
||||
|
||||
from djeuscan.models import Package, Herd, Maintainer, Version
|
||||
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
|
||||
from djeuscan import charts
|
||||
from djeuscan.processing import FakeLogger
|
||||
|
||||
from distutils.version import StrictVersion, LooseVersion
|
||||
|
||||
|
||||
def _compare_versions(version1, version2):
|
||||
try:
|
||||
return cmp(StrictVersion(version1), StrictVersion(version2))
|
||||
# in case of abnormal version number, fall back to LooseVersion
|
||||
except ValueError:
|
||||
return cmp(LooseVersion(version1), LooseVersion(version2))
|
||||
|
||||
|
||||
def _add_safe(storage, key):
|
||||
if key not in storage:
|
||||
storage[key] = 1
|
||||
else:
|
||||
storage[key] += 1
|
||||
|
||||
|
||||
def _add_last_ver(storage, version):
|
||||
key = version['package_id']
|
||||
if key not in storage:
|
||||
storage[key] = version
|
||||
return
|
||||
if version['version'].startswith('9999'):
|
||||
return
|
||||
if _compare_versions(storage[key]['version'],
|
||||
version['version']) < 0:
|
||||
storage[key] = version
|
||||
|
||||
|
||||
@commit_on_success
|
||||
def update_counters(fast=False, nolog=False, logger=None):
|
||||
logger = logger or FakeLogger()
|
||||
|
||||
now = timezone.now()
|
||||
|
||||
categories = {}
|
||||
herds = {}
|
||||
maintainers = {}
|
||||
|
||||
wlog = None
|
||||
|
||||
if not nolog:
|
||||
wlog = WorldLog()
|
||||
wlog.datetime = now
|
||||
|
||||
for cat in Package.objects.values('category').distinct():
|
||||
clog = CategoryLog()
|
||||
clog.datetime = now
|
||||
clog.category = cat['category']
|
||||
categories[clog.category] = clog
|
||||
|
||||
for herd in Herd.objects.all():
|
||||
hlog = HerdLog()
|
||||
hlog.datetime = now
|
||||
hlog.herd = herd
|
||||
herds[herd.id] = hlog
|
||||
|
||||
for maintainer in Maintainer.objects.all():
|
||||
mlog = MaintainerLog()
|
||||
mlog.datetime = now
|
||||
mlog.maintainer = maintainer
|
||||
maintainers[maintainer.id] = mlog
|
||||
|
||||
package_queryset = Package.objects.all()
|
||||
|
||||
n_versions = {}
|
||||
n_packaged = {}
|
||||
n_overlay = {}
|
||||
|
||||
last_versions_gentoo = {}
|
||||
last_versions_overlay = {}
|
||||
last_versions_upstream = {}
|
||||
|
||||
if not fast:
|
||||
attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
|
||||
for version in Version.objects.all().values(*attrs):
|
||||
overlay, packaged = version['overlay'], version['packaged']
|
||||
package_id = version['package_id']
|
||||
|
||||
_add_safe(n_versions, package_id)
|
||||
|
||||
if not packaged:
|
||||
_add_last_ver(last_versions_upstream, version)
|
||||
continue
|
||||
if overlay == 'gentoo':
|
||||
_add_safe(n_packaged, package_id)
|
||||
_add_last_ver(last_versions_gentoo, version)
|
||||
else:
|
||||
_add_safe(n_overlay, package_id)
|
||||
_add_last_ver(last_versions_overlay, version)
|
||||
|
||||
for package in package_queryset.select_related('herds', 'maintainers'):
|
||||
if not fast:
|
||||
package.n_versions = n_versions.get(package.id, 0)
|
||||
package.n_packaged = n_packaged.get(package.id, 0)
|
||||
package.n_overlay = n_overlay.get(package.id, 0)
|
||||
|
||||
default = {'id': None}
|
||||
package.last_version_gentoo_id = last_versions_gentoo.get(
|
||||
package.id, default
|
||||
)['id']
|
||||
package.last_version_overlay_id = last_versions_overlay.get(
|
||||
package.id, default
|
||||
)['id']
|
||||
package.last_version_upstream_id = last_versions_upstream.get(
|
||||
package.id, default
|
||||
)['id']
|
||||
|
||||
package.save()
|
||||
|
||||
n_packages_gentoo = int(package.n_packaged == package.n_versions)
|
||||
n_packages_overlay = int(package.n_overlay and package.n_packaged \
|
||||
+ package.n_overlay == package.n_versions)
|
||||
n_packages_outdated = int(package.n_packaged + package.n_overlay \
|
||||
< package.n_versions)
|
||||
|
||||
def update_row(storage, key):
|
||||
storage[key].n_packages_gentoo += n_packages_gentoo
|
||||
storage[key].n_packages_overlay += n_packages_overlay
|
||||
storage[key].n_packages_outdated += n_packages_outdated
|
||||
|
||||
storage[key].n_versions_gentoo += package.n_packaged
|
||||
storage[key].n_versions_overlay += package.n_overlay
|
||||
storage[key].n_versions_upstream += package.n_versions - \
|
||||
package.n_packaged - \
|
||||
package.n_overlay
|
||||
|
||||
def update_log(storage, qs):
|
||||
for row in qs:
|
||||
update_row(storage, row['id'])
|
||||
|
||||
if not nolog:
|
||||
update_log(herds, package.herds.all().values('id'))
|
||||
update_log(maintainers, package.maintainers.all().values('id'))
|
||||
update_row(categories, package.category)
|
||||
|
||||
wlog.n_packages_gentoo += n_packages_gentoo
|
||||
wlog.n_packages_overlay += n_packages_overlay
|
||||
wlog.n_packages_outdated += n_packages_outdated
|
||||
|
||||
wlog.n_versions_gentoo += package.n_packaged
|
||||
wlog.n_versions_overlay += package.n_overlay
|
||||
wlog.n_versions_upstream += package.n_versions - \
|
||||
package.n_packaged - \
|
||||
package.n_overlay
|
||||
|
||||
if nolog:
|
||||
return
|
||||
|
||||
for clog in categories.values():
|
||||
logger.info('+ [cl] %s' % clog)
|
||||
charts.rrd_update('category-%s' % clog.category, now, clog)
|
||||
clog.save()
|
||||
|
||||
for hlog in herds.values():
|
||||
logger.info('+ [hl] %s' % hlog)
|
||||
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
|
||||
hlog.save()
|
||||
|
||||
for mlog in maintainers.values():
|
||||
logger.info('+ [ml] %s' % mlog)
|
||||
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
|
||||
mlog.save()
|
||||
|
||||
charts.rrd_update('world', now, wlog)
|
||||
wlog.save()
|
||||
108
euscanwww/djeuscan/processing/misc/update_portage_trees.py
Normal file
108
euscanwww/djeuscan/processing/misc/update_portage_trees.py
Normal file
@ -0,0 +1,108 @@
|
||||
import os
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
def _launch_command(cmd, logger=None):
|
||||
"""
|
||||
Helper for launching shell commands inside tasks
|
||||
"""
|
||||
import sys
|
||||
import subprocess
|
||||
import select
|
||||
|
||||
fp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
mask = select.EPOLLIN|select.EPOLLHUP|select.EPOLLERR
|
||||
|
||||
epoll = select.epoll()
|
||||
epoll.register(fp.stdout.fileno(), mask)
|
||||
epoll.register(fp.stderr.fileno(), mask)
|
||||
|
||||
if logger:
|
||||
info, error = logger.info, logger.error
|
||||
else:
|
||||
info = lambda x: sys.stdout.write(x + '\n')
|
||||
error = lambda x: sys.stderr.write(x + '\n')
|
||||
|
||||
try:
|
||||
exited = False
|
||||
while not exited:
|
||||
events = epoll.poll(1)
|
||||
for fileno, event in events:
|
||||
if event & select.EPOLLIN:
|
||||
if fileno == fp.stdout.fileno():
|
||||
source, out = fp.stdout, info
|
||||
else:
|
||||
source, out = fp.stderr, error
|
||||
line = source.readline().rstrip('\n')
|
||||
out("%s[%s]: %s" % (cmd[0], fp.pid, line))
|
||||
elif event & (select.EPOLLERR|select.EPOLLHUP):
|
||||
exited = True
|
||||
finally:
|
||||
epoll.close()
|
||||
|
||||
fp.wait()
|
||||
|
||||
def emerge_sync(logger):
|
||||
"""
|
||||
Launches an emerge --sync
|
||||
"""
|
||||
cmd = ["emerge", "--sync", "--root", settings.PORTAGE_ROOT,
|
||||
"--config-root", settings.PORTAGE_CONFIGROOT]
|
||||
return _launch_command(cmd, logger)
|
||||
|
||||
|
||||
def layman_sync(logger, cache=True):
|
||||
"""
|
||||
Syncs Layman repos
|
||||
"""
|
||||
from layman import Layman
|
||||
import shutil
|
||||
|
||||
l = Layman(config=settings.LAYMAN_CONFIG)
|
||||
|
||||
installed_overlays = l.get_installed()
|
||||
|
||||
for overlay in installed_overlays:
|
||||
logger.info('Cleaning cache for overlay %s...' % overlay)
|
||||
overlay_path = os.path.join(l.config['storage'], overlay)
|
||||
shutil.rmtree(os.path.join(overlay_path, 'metadata/cache'), True)
|
||||
shutil.rmtree(os.path.join(overlay_path, 'metadata/md5-cache'), True)
|
||||
|
||||
# FIXME, try to find a way to log layman output...
|
||||
#l.sync(installed_overlays, output_results=False)
|
||||
cmd = ['layman', '-S', '--config', settings.LAYMAN_CONFIG]
|
||||
_launch_command(cmd, logger)
|
||||
|
||||
cmd = ['egencache', '--jobs', "%s" % settings.EGENCACHE_JOBS,
|
||||
'--rsync', '--config-root', settings.PORTAGE_CONFIGROOT,
|
||||
'--update', '--update-use-local-desc']
|
||||
|
||||
for overlay in installed_overlays:
|
||||
logger.info('Generating cache for overlay %s...' % overlay)
|
||||
overlay_path = os.path.join(l.config['storage'], overlay)
|
||||
if not os.path.exists(os.path.join(overlay_path, 'profiles/repo_name')):
|
||||
continue
|
||||
_launch_command(cmd + ['--repo', overlay], logger)
|
||||
|
||||
def eix_update(logger):
|
||||
"""
|
||||
Launches eix-update
|
||||
"""
|
||||
cmd = ["eix-update"]
|
||||
return _launch_command(cmd, logger)
|
||||
|
||||
|
||||
def update_portage_trees(logger=None):
|
||||
from djeuscan.processing import FakeLogger
|
||||
|
||||
logger = logger or FakeLogger()
|
||||
logger.info("Running emerge --sync")
|
||||
emerge_sync(logger)
|
||||
logger.info("Running layman --sync")
|
||||
layman_sync(logger, cache=True)
|
||||
#logger.info("Running emerge --regen")
|
||||
#emerge_regen()
|
||||
logger.info("Running eix-update")
|
||||
eix_update(logger)
|
||||
logger.info("Done!")
|
||||
Reference in New Issue
Block a user