euscanwww: move processing to scan and misc, enhance update_portage_tree

update_portage_tree() now:
- watch stderr and stdout for each command
- use layman command instead of layman API for sync because
  layman API doesn't work when stdout or stderr is not a real file
  (we could probably work around that with pipes and epoll)
- use egencache instead of emerge to generate cache
- export PORTAGE_CONFIGROOT, ROOT, EIX_CACHEFILE etc.. so they
  are used everywhere

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>

euscanwww/djeuscan/management/commands/regen_rrds.py

@@ -2,11 +2,10 @@ import logging
 from django.core.management.base import BaseCommand
 
 from djeuscan.processing import set_verbosity_level
-from djeuscan.processing.regen_rrds import regen_rrds
+from djeuscan.processing.misc import regen_rrds
 
 logger = logging.getLogger(__name__)
 
-
 class Command(BaseCommand):
     _overlays = {}
     help = 'Regenerate rrd database'

euscanwww/djeuscan/management/commands/scan_metadata.py

@@ -5,7 +5,7 @@ from optparse import make_option
 from django.core.management.base import BaseCommand
 
 from djeuscan.processing import set_verbosity_level
-from djeuscan.processing.scan_metadata import scan_metadata
+from djeuscan.processing.scan import scan_metadata
 
 logger = logging.getLogger(__name__)
 

euscanwww/djeuscan/management/commands/scan_portage.py

@@ -5,7 +5,7 @@ from optparse import make_option
 from django.core.management.base import BaseCommand
 
 from djeuscan.processing import set_verbosity_level
-from djeuscan.processing.scan_portage import scan_portage
+from djeuscan.processing.scan import scan_portage
 
 logger = logging.getLogger(__name__)
 

euscanwww/djeuscan/management/commands/scan_upstream.py

@@ -5,7 +5,7 @@ from optparse import make_option
 from django.core.management.base import BaseCommand
 
 from djeuscan.processing import set_verbosity_level
-from djeuscan.processing.scan_upstream import scan_upstream
+from djeuscan.processing.scan import scan_upstream
 
 logger = logging.getLogger(__name__)
 

euscanwww/djeuscan/management/commands/update_counters.py

@@ -4,7 +4,7 @@ from optparse import make_option
 from django.core.management.base import BaseCommand
 
 from djeuscan.processing import set_verbosity_level
-from djeuscan.processing.update_counters import update_counters
+from djeuscan.processing.misc import update_counters
 
 logger = logging.getLogger(__name__)
 

euscanwww/djeuscan/management/commands/update_portage_trees.py

@@ -0,0 +1,16 @@
+import logging
+from django.core.management.base import BaseCommand
+
+from djeuscan.processing import set_verbosity_level
+from djeuscan.processing.misc import update_portage_trees
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    _overlays = {}
+    help = 'Regenerate rrd database'
+
+    def handle(self, *args, **options):
+        set_verbosity_level(logger, options.get("verbosity", 1))
+        update_portage_trees(logger=logger)

euscanwww/djeuscan/processing/__init__.py

@@ -1,5 +1,3 @@
-import logging
-
 
 class FakeLogger(object):
     def __getattr__(self, key):
@@ -7,6 +5,8 @@ class FakeLogger(object):
 
 
 def set_verbosity_level(logger, verbosity):
+    import logging
+
     try:
         verbosity = int(verbosity)
     except (ValueError, TypeError):
@@ -29,3 +29,4 @@ def set_verbosity_level(logger, verbosity):
     logger.setLevel(levels[verbosity])
 
     return logger
+

euscanwww/djeuscan/processing/misc/__init__.py

@@ -0,0 +1,5 @@
+__all__ = ["regen_rrds", "update_counters", "update_portage_trees"]
+
+from regen_rrds import regen_rrds
+from update_counters import update_counters
+from update_portage_trees import update_portage_trees

euscanwww/djeuscan/processing/misc/regen_rrds.py

@@ -3,7 +3,6 @@ from djeuscan import charts
 
 from djeuscan.processing import FakeLogger
 
-
 def regen_rrds(logger=None):
     """
     Regenerates the rrd database

euscanwww/djeuscan/processing/misc/update_counters.py

@@ -157,17 +157,17 @@ def update_counters(fast=False, nolog=False, logger=None):
         return
 
     for clog in categories.values():
-        logger.info('+ [cl] %s\n' % clog)
+        logger.info('+ [cl] %s' % clog)
         charts.rrd_update('category-%s' % clog.category, now, clog)
         clog.save()
 
     for hlog in herds.values():
-        logger.info('+ [hl] %s\n' % hlog)
+        logger.info('+ [hl] %s' % hlog)
         charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
         hlog.save()
 
     for mlog in maintainers.values():
-        logger.info('+ [ml] %s\n' % mlog)
+        logger.info('+ [ml] %s' % mlog)
         charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
         mlog.save()
 

euscanwww/djeuscan/processing/misc/update_portage_trees.py

@@ -0,0 +1,108 @@
+import os
+
+from django.conf import settings
+
+def _launch_command(cmd, logger=None):
+    """
+    Helper for launching shell commands inside tasks
+    """
+    import sys
+    import subprocess
+    import select
+
+    fp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    mask = select.EPOLLIN|select.EPOLLHUP|select.EPOLLERR
+
+    epoll = select.epoll()
+    epoll.register(fp.stdout.fileno(), mask)
+    epoll.register(fp.stderr.fileno(), mask)
+
+    if logger:
+        info, error = logger.info, logger.error
+    else:
+        info = lambda x: sys.stdout.write(x + '\n')
+        error = lambda x: sys.stderr.write(x + '\n')
+
+    try:
+        exited = False
+        while not exited:
+            events = epoll.poll(1)
+            for fileno, event in events:
+                if event & select.EPOLLIN:
+                    if fileno == fp.stdout.fileno():
+                        source, out = fp.stdout, info
+                    else:
+                        source, out = fp.stderr, error
+                    line = source.readline().rstrip('\n')
+                    out("%s[%s]: %s" % (cmd[0], fp.pid, line))
+                elif event & (select.EPOLLERR|select.EPOLLHUP):
+                    exited = True
+    finally:
+        epoll.close()
+
+    fp.wait()
+
+def emerge_sync(logger):
+    """
+    Launches an emerge --sync
+    """
+    cmd = ["emerge", "--sync", "--root", settings.PORTAGE_ROOT,
+           "--config-root", settings.PORTAGE_CONFIGROOT]
+    return _launch_command(cmd, logger)
+
+
+def layman_sync(logger, cache=True):
+    """
+    Syncs Layman repos
+    """
+    from layman import Layman
+    import shutil
+
+    l = Layman(config=settings.LAYMAN_CONFIG)
+
+    installed_overlays = l.get_installed()
+
+    for overlay in installed_overlays:
+        logger.info('Cleaning cache for overlay %s...' % overlay)
+        overlay_path = os.path.join(l.config['storage'], overlay)
+        shutil.rmtree(os.path.join(overlay_path, 'metadata/cache'), True)
+        shutil.rmtree(os.path.join(overlay_path, 'metadata/md5-cache'), True)
+
+    # FIXME, try to find a way to log layman output...
+    #l.sync(installed_overlays, output_results=False)
+    cmd = ['layman', '-S', '--config', settings.LAYMAN_CONFIG]
+    _launch_command(cmd, logger)
+
+    cmd = ['egencache', '--jobs', "%s" % settings.EGENCACHE_JOBS,
+           '--rsync', '--config-root', settings.PORTAGE_CONFIGROOT,
+           '--update', '--update-use-local-desc']
+
+    for overlay in installed_overlays:
+        logger.info('Generating cache for overlay %s...' % overlay)
+        overlay_path = os.path.join(l.config['storage'], overlay)
+        if not os.path.exists(os.path.join(overlay_path, 'profiles/repo_name')):
+            continue
+        _launch_command(cmd + ['--repo', overlay], logger)
+
+def eix_update(logger):
+    """
+    Launches eix-update
+    """
+    cmd = ["eix-update"]
+    return _launch_command(cmd, logger)
+
+
+def update_portage_trees(logger=None):
+    from djeuscan.processing import FakeLogger
+
+    logger = logger or FakeLogger()
+    logger.info("Running emerge --sync")
+    emerge_sync(logger)
+    logger.info("Running layman --sync")
+    layman_sync(logger, cache=True)
+    #logger.info("Running emerge --regen")
+    #emerge_regen()
+    logger.info("Running eix-update")
+    eix_update(logger)
+    logger.info("Done!")

euscanwww/djeuscan/processing/scan/__init__.py

@@ -0,0 +1,7 @@
+__all__ = [
+    "scan_metadata", "scan_portage", "scan_upstream",
+]
+
+from scan_metadata import scan_metadata
+from scan_portage import scan_portage
+from scan_upstream import scan_upstream

euscanwww/djeuscan/processing/scan/scan_upstream.py

@@ -11,8 +11,9 @@ from djeuscan.models import Package, Version, EuscanResult, VersionLog
 
 
 class ScanUpstream(object):
-    def __init__(self, logger=None):
+    def __init__(self, logger=None, purge_versions=False):
         self.logger = logger or FakeLogger()
+        self.purge_versions = purge_versions
 
     def scan(self, package):
         CONFIG["format"] = "dict"
@@ -31,23 +32,20 @@ class ScanUpstream(object):
         except KeyError:
             return {}
 
-        with commit_on_success():
+        obj = self.store_package(cpv)
-            obj = self.store_package(cpv)
 
-            for res in out[package]["result"]:
+        for res in out[package]["result"]:
-                self.store_version(
+            self.store_version(
-                    obj,
+                obj,
-                    res["version"],
+                res["version"],
-                    " ".join(res["urls"]),
+                " ".join(res["urls"]),
-                    res["type"],
+                res["type"],
-                    res["handler"],
+                res["handler"],
-                    res["confidence"],
+                res["confidence"],
-                )
+            )
 
             self.store_result(obj, out_json, scan_time, ebuild)
 
-        return out
-
     def store_result(self, package, formatted_log, scan_time, ebuild):
         # Remove previous logs
         EuscanResult.objects.filter(package=package).delete()
@@ -70,7 +68,8 @@ class ScanUpstream(object):
 
         # Set all versions dead, then set found versions alive and
         # delete old versions
-        Version.objects.filter(package=obj, packaged=False).update(alive=False)
+        if self.purge_versions:
+            Version.objects.filter(package=obj, packaged=False).update(alive=False)
 
         return obj
 
@@ -111,33 +110,34 @@ class ScanUpstream(object):
         package.save()
 
 
+    def purge_old_versions(self):
+        if not self.purge_versions:
+            return
+
+        versions = Version.objects.filter(packaged=False, alive=False)
+        for version in versions:
+            VersionLog.objects.create(
+                package=version.package,
+                action=VersionLog.VERSION_REMOVED,
+                slot=version.slot,
+                revision=version.revision,
+                version=version.version,
+                overlay=version.overlay
+            )
+
+            version.package.n_versions -= 1
+            version.package.save()
+
+            self.logger.info('- [u] %s %s' % (version, version.urls))
+
+        versions.delete()
+
 @commit_on_success
-def do_purge_versions(logger=None):
-    logger = logger or FakeLogger()
-
-    # For each dead versions
-    for version in Version.objects.filter(packaged=False, alive=False):
-        VersionLog.objects.create(
-            package=version.package,
-            action=VersionLog.VERSION_REMOVED,
-            slot=version.slot,
-            revision=version.revision,
-            version=version.version,
-            overlay=version.overlay
-        )
-
-        version.package.n_versions -= 1
-        version.package.save()
-
-        logger.info('- [u] %s %s' % (version, version.urls))
-    Version.objects.filter(packaged=False, alive=False).delete()
-
-
 def scan_upstream(packages=None, purge_versions=False,
                   logger=None):
     logger = logger or FakeLogger()
 
-    scan_handler = ScanUpstream(logger=logger)
+    scan_handler = ScanUpstream(logger=logger, purge_versions=purge_versions)
 
     logger.info('Scanning upstream...')
 
@@ -148,14 +148,10 @@ def scan_upstream(packages=None, purge_versions=False,
 
     for pkg in packages:
         if isinstance(pkg, Package):
-            curr = scan_handler.scan('%s/%s' % (pkg.category, pkg.name))
+            scan_handler.scan('%s/%s' % (pkg.category, pkg.name))
         else:
-            curr = scan_handler.scan(pkg)
+            scan_handler.scan(pkg)
-        if not curr:
-            result = False
 
-    if purge_versions:
+    scan_handler.purge_old_versions()
-        do_purge_versions(logger=logger)
 
     logger.info('Done.')
-    return result

euscanwww/djeuscan/processing/update_portage_trees.py

@@ -1,66 +0,0 @@
-import subprocess
-from StringIO import StringIO
-
-from django.conf import settings
-
-from djeuscan.processing import FakeLogger
-
-
-def _launch_command(cmd):
-    """
-    Helper for launching shell commands inside tasks
-    """
-    fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                          stderr=subprocess.PIPE)
-    output = StringIO(fp.communicate()[0])
-    return output.getvalue()
-
-
-def emerge_sync():
-    """
-    Launches an emerge --sync
-    """
-    cmd = ["emerge", "--sync", "--root", settings.PORTAGE_ROOT,
-           "--config-root", settings.PORTAGE_CONFIGROOT]
-    return _launch_command(cmd)
-
-
-def layman_sync():
-    """
-    Syncs Layman repos
-    """
-    from layman import Layman
-    l = Layman(config=settings.LAYMAN_CONFIG)
-    return l.sync(l.get_installed(), output_results=False)
-
-
-def emerge_regen():
-    """
-    Launches emerge --regen
-    """
-    cmd = [
-        "emerge", "--regen", "--jobs", settings.EMERGE_REGEN_JOBS, "--root",
-        settings.PORTAGE_ROOT, "--config-root", settings.PORTAGE_CONFIGROOT
-    ]
-    return _launch_command(cmd)
-
-
-def eix_update():
-    """
-    Launches eix-update
-    """
-    cmd = ["eix-update"]
-    return _launch_command(cmd)
-
-
-def update_portage_trees(logger=None):
-    logger = logger or FakeLogger()
-    logger.info("Running emerge --sync")
-    emerge_sync()
-    logger.info("Running layman --sync")
-    layman_sync()
-    logger.info("Running emerge --regen")
-    emerge_regen()
-    logger.info("Running eix-update")
-    eix_update()
-    logger.info("Done!")

euscanwww/euscanwww/settings.py

@@ -222,10 +222,10 @@ ACCOUNT_ACTIVATION_DAYS = 7
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 
 # djeuscan tasks
-PORTAGE_ROOT = "/usr/portage/"
+PORTAGE_ROOT = "/"
-PORTAGE_CONFIGROOT = PORTAGE_ROOT
+PORTAGE_CONFIGROOT = "/"
 LAYMAN_CONFIG = "/etc/layman/layman.cfg"
-EMERGE_REGEN_JOBS = 4
+EGENCACHE_JOBS = 4
 
 # Celery config
 import djcelery
@@ -255,3 +255,7 @@ except ImportError, ex:
         "settings.py: error importing local settings file:\n"
         "\t%s\nDo you have a local_settings.py module?\n" % str(ex)
     )
+
+os.environ['ROOT'] = PORTAGE_ROOT
+os.environ['PORTAGE_CONFIGROOT'] = PORTAGE_CONFIGROOT
+os.environ['EIX_CACHEFILE'] = os.path.join(PORTAGE_ROOT, 'var/cache/eix')