euscanwww: add --category to scan_metadata, remove homepage and description

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>

euscanwww/djeuscan/management/commands/scan_metadata.py

@@ -19,6 +19,11 @@ class Command(BaseCommand):
             dest='all',
             default=False,
             help='Scan all packages'),
+        make_option('--category',
+            action='store',
+            dest='category',
+            default=None,
+            help='Scan only this category'),
         )
     args = '<package package ...>'
     help = 'Scans metadata and fills database'
@@ -26,7 +31,7 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         set_verbosity_level(logger, options.get("verbosity", 1))
 
-        if options['all']:
+        if options['all'] or options['category']:
             packages = None
 
         elif len(args):
@@ -34,4 +39,8 @@ class Command(BaseCommand):
         else:
             packages = [pkg[:-1] for pkg in sys.stdin.readlines()]
 
-        scan_metadata(packages=packages, logger=logger)
+        scan_metadata(
+            packages=packages,
+            category=options['category'],
+            logger=logger
+        )

euscanwww/djeuscan/processing/scan_metadata.py

@@ -1,3 +1,5 @@
+import os.path
+
 from gentoolkit.query import Query
 from gentoolkit.errors import GentoolkitFatalError
 
@@ -8,17 +10,19 @@ from django.core.exceptions import ValidationError
 from djeuscan.models import Package, Herd, Maintainer
 from djeuscan.processing import FakeLogger
 
-
 class ScanMetadata(object):
     def __init__(self, logger=None):
         self.style = color_style()
         self.logger = logger or FakeLogger()
 
-    @commit_on_success
     def scan(self, query=None, obj=None):
-        matches = Query(query).find(
+        matches = Query(query).smart_find(
-            include_masked=True,
+                in_installed=True,
-            in_installed=False,
+                in_porttree=True,
+                in_overlay=True,
+                include_masked=True,
+                show_progress=False,
+                no_matches_fatal=False,
         )
 
         if not matches:
@@ -39,65 +43,52 @@ class ScanMetadata(object):
         else:
             created = False
 
-        try:
-            obj.homepage = pkg.environment("HOMEPAGE")
-            obj.description = pkg.environment("DESCRIPTION")
-        except GentoolkitFatalError, err:
-            self.logger.error(
-                self.style.ERROR(
-                    "Gentoolkit fatal error: '%s'" % str(err)
-                )
-            )
-
         if created:
             self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name))
 
-        if pkg.metadata:
+        if not pkg.metadata:
-            herds = dict(
+            return
-                [(herd[0], herd) for herd in pkg.metadata.herds(True)]
-            )
-            maintainers = dict(
-                [(m.email, m) for m in pkg.metadata.maintainers()]
-            )
 
-            existing_herds = [h.herd for h in obj.herds.all()]
+        herds = dict(
-            new_herds = set(herds.keys()).difference(existing_herds)
+            [(herd[0], herd) for herd in pkg.metadata.herds(True)]
-            old_herds = set(existing_herds).difference(herds.keys())
+        )
+        maintainers = dict(
+            [(m.email, m) for m in pkg.metadata.maintainers()]
+        )
 
-            existing_maintainers = [m.email for m in obj.maintainers.all()]
+        existing_herds = [h.herd for h in obj.herds.all()]
-            new_maintainers = set(
+        new_herds = set(herds.keys()).difference(existing_herds)
-                maintainers.keys()).difference(existing_maintainers
+        old_herds = set(existing_herds).difference(herds.keys())
-            )
-            old_maintainers = set(
-                existing_maintainers).difference(maintainers.keys()
-            )
 
-            for herd in obj.herds.all():
+        existing_maintainers = [m.email for m in obj.maintainers.all()]
-                if herd.herd in old_herds:
+        new_maintainers = set(maintainers.keys()).difference(existing_maintainers)
-                    obj.herds.remove(herd)
+        old_maintainers = set(existing_maintainers).difference(maintainers.keys())
 
-            for herd in new_herds:
+        for herd in obj.herds.all():
-                herd = self.store_herd(*herds[herd])
+            if herd.herd in old_herds:
-                obj.herds.add(herd)
+                obj.herds.remove(herd)
 
-            for maintainer in obj.maintainers.all():
+        for herd in new_herds:
-                if maintainer.email in old_maintainers:
+            herd = self.store_herd(*herds[herd])
-                    obj.maintainers.remove(maintainer)
+            obj.herds.add(herd)
 
-            for maintainer in new_maintainers:
+        for maintainer in obj.maintainers.all():
-                maintainer = maintainers[maintainer]
+            if maintainer.email in old_maintainers:
-                try:
+                obj.maintainers.remove(maintainer)
-                    maintainer = self.store_maintainer(
+
-                        maintainer.name, maintainer.email
+        for maintainer in new_maintainers:
+            maintainer = maintainers[maintainer]
+            try:
+                maintainer = self.store_maintainer(
+                    maintainer.name, maintainer.email
                     )
-                    obj.maintainers.add(maintainer)
+                obj.maintainers.add(maintainer)
-                except ValidationError:
+            except ValidationError:
-                    self.logger.error(
+                self.logger.error(
-                        self.style.ERROR("Bad maintainer: '%s' '%s'" % \
+                    self.style.ERROR("Bad maintainer: '%s' '%s'" % \
                                          (maintainer.name, maintainer.email))
                     )
         obj.save()
-        return True
 
     def store_herd(self, name, email):
         if not name:
@@ -134,15 +125,18 @@ class ScanMetadata(object):
             )
         return maintainer
 
-
+@commit_on_success
-def scan_metadata(packages=None, logger=None):
+def scan_metadata(packages=None, category=None, logger=None):
     scan_handler = ScanMetadata(logger=logger)
-    if not packages:
+
+    if category:
+        packages = Package.objects.filter(category=category)
+    elif not packages:
         packages = Package.objects.all()
 
     for pkg in packages:
         if isinstance(pkg, Package):
-            result = scan_handler.scan('%s/%s' % (pkg.category, pkg.name), pkg)
+            scan_handler.scan('%s/%s' % (pkg.category, pkg.name), pkg)
         else:
-            result = scan_handler.scan(pkg)
+            scan_handler.scan(pkg)
-    return result
+

euscanwww/djeuscan/processing/scan_portage.py

@@ -302,4 +302,4 @@ def scan_portage(packages=None, category=None, no_log=False, purge_packages=Fals
                 scan_handler.scan(pkg)
 
     logger.info('Done.')
-    return True
+