Merge branch 'master' of https://github.com/iksaif/euscan

Conflicts:
	euscanwww/djeuscan/processing/scan_portage.py

Signed-off-by: volpino <fox91@anche.no>

euscanwww/djeuscan/management/commands/scan_portage.py

@@ -49,7 +49,6 @@ class Command(BaseCommand):
 
         if options['all']:
             packages = None
-
         elif len(args):
             packages = [pkg for pkg in args]
         else:

euscanwww/djeuscan/processing/scan_portage.py

@@ -1,7 +1,6 @@
 import subprocess
 import portage
-import os
+
-import re
 from xml.etree.ElementTree import iterparse, ParseError
 
 from django.db.transaction import commit_on_success
@@ -15,11 +14,11 @@ from djeuscan.models import Package, Version, VersionLog
 
 class ScanPortage(object):
     def __init__(self, logger=None, no_log=False, purge_packages=False,
-                 kill_versions=False):
+                 purge_versions=False):
         self.logger = logger or FakeLogger()
         self.no_log = no_log
         self.purge_packages = purge_packages
-        self.kill_versions = kill_versions
+        self.purge_versions = purge_versions
 
         self.style = color_style()
         self._cache = {'packages': {}, 'versions': {}}
@@ -57,47 +56,28 @@ class ScanPortage(object):
         )
         self._cache['versions'][key] = version
 
-    def overlays(self):
-        if self._overlays:
-            return self._overlays
-
-        env = os.environ
-        env['OVERLAYS_LIST'] = 'all'
-        env['PRINT_COUNT_ALWAYS'] = 'never'
-
-        cmd = ['eix', '-!']
-
-        output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
-            communicate()[0]
-        output = output.strip().strip('\n').split('\n')
-
-        overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
-
-        self._overlays = {}
-
-        for line in output:
-            match = overlay_re.match(line)
-            if not match:
-                continue
-            self._overlays[match.group('key')] = match.group('name')
-
-        return self._overlays
-
     def scan(self, query=None):
-        if self.purge_packages:
+        cmd = ['eix', '--xml']
-            with commit_on_success():
-                for package in Package.objects.all():
-                    self.logger.info('- [p] %s' % (package))
-                    package.delete()
-
-        cmd = ['eix', '--xml', '--pure-packages', '-x']
         if query:
             cmd.extend(['--exact', query])
 
-        if self.kill_versions:
+        if not query:
-            self.logger.info('Killing existing versions...')
+            current_packages = Package.objects.all()
-            Version.objects.filter(packaged=True).update(alive=False)
+        elif '/' in query:
-            self.logger.info('done')
+            cat, pkg = portage.catsplit(query)
+            current_packages = Package.objects.filter(category=cat, name=pkg)
+        else:
+            current_packages = Package.objects.filter(name=query)
+
+        if self.purge_versions:
+            if not query:
+                self.logger.info('Killing existing versions...')
+                Version.objects.filter(packaged=True).update(alive=False)
+                self.logger.info('done')
+            else:
+                for package in current_packages:
+                    Version.objects.filter(package=package, packaged=True).\
+                                    update(alive=False)
 
         sub = subprocess.Popen(cmd, stdout=subprocess.PIPE)
 
@@ -122,10 +102,11 @@ class ScanPortage(object):
                         "Unknown package '%s'" % query
                     )
                 )
-            return
+                return
 
         cat, pkg, homepage, desc = ("", "", "", "")
         versions = []
+        packages_alive = set()
 
         for event, elem in parser:
             if event == "start":  # on tag opening
@@ -149,6 +130,7 @@ class ScanPortage(object):
                     # package tag has been closed, saving everything!
                     with commit_on_success():
                         package = self.store_package(cat, pkg, homepage, desc)
+                        packages_alive.add('%s/%s' % (cat, pkg))
                         for cpv, slot, overlay in versions:
                             self.store_version(package, cpv, slot, overlay)
 
@@ -160,6 +142,15 @@ class ScanPortage(object):
                     # clean old data
                     cat = ""
 
+        if self.purge_packages:
+            for package in current_packages:
+                cp = "%s/%s" % (package.category, package.name)
+                if cp not in packages_alive:
+                    self.logger.info('- [p] %s' % (package))
+                    package.delete()
+        if self.purge_versions:
+            self.purge_old_versions(current_packages)
+
     def store_package(self, cat, pkg, homepage, description):
         created = False
         obj = self.cache_get_package(cat, pkg)
@@ -178,7 +169,7 @@ class ScanPortage(object):
 
         # Set all versions dead, then set found versions alive and
         # delete old versions
-        if not self.kill_versions:
+        if not self.purge_versions:
             Version.objects.filter(
                 package=obj,
                 packaged=True
@@ -188,12 +179,7 @@ class ScanPortage(object):
 
     def store_version(self, package, cpv, slot, overlay):
         cat, pkg, ver, rev = portage.catpkgsplit(cpv)
-
+        if not overlay:
-        overlays = self.overlays()
-
-        if overlay in overlays:
-            overlay = overlays[overlay]
-        else:
             overlay = 'gentoo'
 
         created = False
@@ -247,51 +233,57 @@ class ScanPortage(object):
             overlay=obj.overlay
         )
 
+    def purge_old_versions(self, packages):
+        # For each dead versions
+        if packages:
+            versions = []
+            for package in packages:
+                qs = Version.objects.filter(package=package, packaged=True,
+                                            alive=False)
+                for version in qs:
+                    versions.append(version)
+        else:
+            versions = Version.objects.filter(packaged=True, alive=False)
+
+        for version in versions:
+            if version.overlay == 'gentoo':
+                version.package.n_packaged -= 1
+            else:
+                version.package.n_overlay -= 1
+            version.package.n_versions -= 1
+            version.package.save()
+
+            self.logger.info('- [v] %s' % (version))
+
+            if self.no_log:
+                continue
+
+            VersionLog.objects.create(
+                package=version.package,
+                action=VersionLog.VERSION_REMOVED,
+                slot=version.slot,
+                revision=version.revision,
+                version=version.version,
+                overlay=version.overlay
+            )
+
+        Version.objects.filter(packaged=True, alive=False).delete()
+
 
 @commit_on_success
-def do_purge_versions(logger=None, no_log=False):
-    logger = logger or FakeLogger()
-
-    # For each dead versions
-    for version in Version.objects.filter(packaged=True, alive=False):
-        if version.overlay == 'gentoo':
-            version.package.n_packaged -= 1
-        else:
-            version.package.n_overlay -= 1
-        version.package.n_versions -= 1
-        version.package.save()
-
-        logger.info('- [v] %s' % (version))
-
-        if no_log:
-            continue
-
-        VersionLog.objects.create(
-            package=version.package,
-            action=VersionLog.VERSION_REMOVED,
-            slot=version.slot,
-            revision=version.revision,
-            version=version.version,
-            overlay=version.overlay
-        )
-
-    Version.objects.filter(packaged=True, alive=False).delete()
-
-
 def scan_portage(packages=None, no_log=False, purge_packages=False,
                  purge_versions=False, prefetch=False, logger=None):
 
     logger = logger or FakeLogger()
-    kill_versions = False
+
     if packages is None:
         prefetch = True
-        kill_versions = True
 
     scan_handler = ScanPortage(
         logger=logger,
         no_log=no_log,
         purge_packages=purge_packages,
-        kill_versions=kill_versions,
+        purge_versions=purge_versions,
     )
 
     logger.info('Scanning portage tree...')
@@ -313,7 +305,5 @@ def scan_portage(packages=None, no_log=False, purge_packages=False,
             else:
                 scan_handler.scan(pkg)
 
-    if purge_versions:
-        do_purge_versions(logger=logger, no_log=no_log)
     logger.info('Done.')
     return True