euscanwww: implement scan-upstream, fix other commands

Also break migrations :).

Signed-off-by: Corentin Chary <corentincj@iksaif.net>

euscanwww/euscan/management/commands/scan-metadata.py

@@ -12,7 +12,7 @@ from django.core.management.base import BaseCommand, CommandError
 from euscanwww.euscan.models import Package, Herd, Maintainer
 
 from gentoolkit.query import Query
-from gentoolkit.eclean.search import (port_settings)
+from gentoolkit.errors import GentoolkitFatalError
 
 class Command(BaseCommand):
     _overlays = {}
@@ -37,7 +37,7 @@ class Command(BaseCommand):
             raise CommandError('You must specify a package or use --all')
 
         if not options['quiet']:
-            self.stdout.write('Scanning portage tree...\n')
+            self.stdout.write('Scanning metadata...\n')
 
         if len(args) == 0:
             for pkg in Package.objects.all():
@@ -62,21 +62,25 @@ class Command(BaseCommand):
 
 	matches = sorted(matches)
         pkg = matches.pop()
-	if pkg.version == '9999':
+	if pkg.version == '9999' and len(matches):
 		pkg = matches.pop()
 
         obj, created = Package.objects.get_or_create(category=pkg.category, name=pkg.name)
 
-        obj.homepage = pkg.environment("HOMEPAGE")
-        obj.description = pkg.environment("DESCRIPTION")
+        try:
+            obj.homepage = pkg.environment("HOMEPAGE")
+            obj.description = pkg.environment("DESCRIPTION")
+        except GentoolkitFatalError, err:
+            sys.stderr.write(self.style.ERROR("Gentoolkit fatal error: '%s'\n" % str(err)))
 
-        for herd in pkg.metadata.herds(True):
-            herd = self.store_herd(options, herd[0], herd[1])
-            obj.herds.add(herd)
+        if pkg.metadata:
+            for herd in pkg.metadata.herds(True):
+                herd = self.store_herd(options, herd[0], herd[1])
+                obj.herds.add(herd)
 
-        for maintainer in pkg.metadata.maintainers():
-            maintainer = self.store_maintainer(options, maintainer.name, maintainer.email)
-            obj.maintainers.add(maintainer)
+            for maintainer in pkg.metadata.maintainers():
+                maintainer = self.store_maintainer(options, maintainer.name, maintainer.email)
+                obj.maintainers.add(maintainer)
 
         if not options['quiet']:
             sys.stdout.write('[p] %s/%s\n' % (pkg.category, pkg.name))

euscanwww/euscan/management/commands/scan-portage.py

@@ -36,9 +36,6 @@ class Command(BaseCommand):
         if not options['quiet']:
             self.stdout.write('Scanning portage tree...\n')
 
-        Version.objects.all().delete()
-        Package.objects.all().delete()
-
         for package in args:
             self.scan(options, package)
 
@@ -119,11 +116,13 @@ class Command(BaseCommand):
         if created:
             if not options['quiet']:
                 sys.stdout.write('[p] %s/%s\n' % (cat, pkg))
-            obj.save()
 
         # Delete previous versions to handle incremental scan correctly
         Version.objects.filter(package=obj, packaged=True).delete()
 
+        obj.n_versions = Version.objects.filter(package=obj).count()
+        obj.save()
+
         return obj
 
     def store_version(self, options, package, cpv, slot, overlay):
@@ -143,5 +142,8 @@ class Command(BaseCommand):
                                                      revision=rev, version=ver,
                                                      overlay=overlay)
         obj.packaged = True
-        #obj.save()
+        obj.save()
 
+        package.n_versions += 1
+        package.n_packaged += 1
+        package.save()

euscanwww/euscan/management/commands/scan-upstream.py

@@ -0,0 +1,147 @@
+import subprocess
+import portage
+import sys
+import os
+import re
+
+from datetime import datetime
+from portage import versions
+from optparse import make_option
+
+from django.db.transaction import commit_on_success
+from django.core.management.base import BaseCommand, CommandError
+from euscanwww.euscan.models import Package, Version, EuscanResult
+
+class Command(BaseCommand):
+    _overlays = {}
+
+    option_list = BaseCommand.option_list + (
+        make_option('--all',
+            action='store_true',
+            dest='all',
+            default=False,
+            help='Scan all packages'),
+        make_option('--parallel',
+            action='store_true',
+            dest='parallel',
+            default=False,
+            help='Use GNU Parallel'),
+        make_option('--quiet',
+            action='store_true',
+            dest='quiet',
+            default=False,
+            help='Be quiet'),
+        )
+    args = '<package package ...>'
+    help = 'Scans metadata and fills database'
+
+    def handle(self, *args, **options):
+        if len(args) == 0 and options['all'] == False:
+            raise CommandError('You must specify a package or use --all')
+
+        if not options['quiet']:
+            self.stdout.write('Scanning upstream...\n')
+
+        packages = []
+
+        if len(args) == 0:
+            for pkg in Package.objects.all():
+                packages.append('%s/%s' % (pkg.category, pkg.name))
+        else:
+            packages = list(args)
+
+        self.scan(options, packages)
+
+        if not options['quiet']:
+            self.stdout.write('Done.\n')
+
+    @commit_on_success
+    def scan(self, options, packages=None):
+        if options['parallel']:
+            jobs = '\n'.join(packages)
+            cmd = ['gparallel', '--jobs', '150%', 'euscan']
+
+            fp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+            output = fp.communicate(jobs)[0]
+
+            self.parse_output(options, output)
+        else:
+            for package in packages:
+                cmd = ['euscan', package]
+
+                fp = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+                output = fp.communicate()[0]
+
+                self.parse_output(options, output)
+
+    def parse_output(self, options, output):
+        from portage.versions import _cp
+
+        package_re = re.compile(r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$')
+        version_re = re.compile(r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$')
+
+        package = None
+        log = ""
+
+        for line in output.split('\n'):
+            match = package_re.match(line)
+            if match:
+                if package:
+                    self.store_result(options, package, log)
+
+                cpv = match.group('cpv')
+                package = self.store_package(options, cpv)
+                log = line
+                continue
+
+            log += line
+
+            match = version_re.match(line)
+            if match:
+                ver = match.group('ver')
+                url = match.group('url')
+                self.store_version(options, package, ver, url)
+
+        if package:
+            self.store_result(options, package, log)
+
+    def store_result(self, options, package, log):
+        obj = EuscanResult()
+        obj.package = package
+        obj.result = log
+        obj.datetime = datetime.now()
+        obj.save()
+
+    def store_package(self, options, cpv):
+        cat, pkg, ver, rev = portage.catpkgsplit(cpv)
+
+        obj, created = Package.objects.get_or_create(category=cat, name=pkg)
+
+        if created:
+            if not options['quiet']:
+                sys.stdout.write('[p] %s/%s\n' % (cat, pkg))
+
+        # Delete previous versions to handle incremental scan correctly
+        Version.objects.filter(package=obj, packaged=False).delete()
+
+        obj.n_versions = Version.objects.filter(package=obj).count()
+        obj.save()
+
+        return obj
+
+    def store_version(self, options, package, ver, url):
+        obj, created = Version.objects.get_or_create(package=package, slot='',
+                                                     revision='r0', version=ver,
+                                                     overlay='')
+
+        if created or not obj.packaged:
+            if not options['quiet']:
+                sys.stdout.write('[u] %s/%s-%s %s\n' % (package.category, package.name,
+                                                        ver, url))
+
+            obj.url = url
+            obj.packaged = False
+            obj.save()
+
+        package.n_versions += 1
+        package.save()