euscanwwww: implement scan-portage and scan-metadata

Signed-off-by: Corentin Chary <corentincj@iksaif.net>

euscanwww/euscan/management/commands/scan-metadata.py

@@ -0,0 +1,110 @@
+import subprocess
+import portage
+import sys
+import os
+import re
+
+from portage import versions
+from optparse import make_option
+
+from django.db.transaction import commit_on_success
+from django.core.management.base import BaseCommand, CommandError
+from euscanwww.euscan.models import Package, Herd, Maintainer
+
+from gentoolkit.query import Query
+from gentoolkit.eclean.search import (port_settings)
+
+class Command(BaseCommand):
+    _overlays = {}
+
+    option_list = BaseCommand.option_list + (
+        make_option('--all',
+            action='store_true',
+            dest='all',
+            default=False,
+            help='Scan all packages'),
+        make_option('--quiet',
+            action='store_true',
+            dest='quiet',
+            default=False,
+            help='Be quiet'),
+        )
+    args = '<package package ...>'
+    help = 'Scans metadata and fills database'
+
+    def handle(self, *args, **options):
+        if len(args) == 0 and options['all'] == False:
+            raise CommandError('You must specify a package or use --all')
+
+        if not options['quiet']:
+            self.stdout.write('Scanning portage tree...\n')
+
+        if len(args) == 0:
+            for pkg in Package.objects.all():
+                self.scan(options, '%s/%s' % (pkg.category, pkg.name))
+        else:
+            for package in args:
+                self.scan(options, package)
+
+        if not options['quiet']:
+            self.stdout.write('Done.\n')
+
+    @commit_on_success
+    def scan(self, options, query=None):
+        matches = Query(query).find(
+                include_masked=True,
+                in_installed=False
+        )
+
+        if not matches:
+                sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % query))
+                return
+
+	matches = sorted(matches)
+        pkg = matches.pop()
+	if pkg.version == '9999':
+		pkg = matches.pop()
+
+        obj, created = Package.objects.get_or_create(category=pkg.category, name=pkg.name)
+
+        obj.homepage = pkg.environment("HOMEPAGE")
+        obj.description = pkg.environment("DESCRIPTION")
+
+        for herd in pkg.metadata.herds(True):
+            herd = self.store_herd(options, herd[0], herd[1])
+            obj.herds.add(herd)
+
+        for maintainer in pkg.metadata.maintainers():
+            maintainer = self.store_maintainer(options, maintainer.name, maintainer.email)
+            obj.maintainers.add(maintainer)
+
+        if not options['quiet']:
+            sys.stdout.write('[p] %s/%s\n' % (pkg.category, pkg.name))
+
+        obj.save()
+
+    def store_herd(self, options, name, email):
+        herd, created = Herd.objects.get_or_create(herd=name)
+
+        if created or herd.email != email:
+            if not options['quiet']:
+                sys.stdout.write('[h] %s <%s>\n' % (name, email))
+
+            herd.email = email
+            herd.save()
+
+        return herd
+
+    def store_maintainer(self, options, name, email):
+        if not name:
+            name = email
+
+        maintainer, created = Maintainer.objects.get_or_create(name=name, email=email)
+
+        if created:
+            if not options['quiet']:
+                sys.stdout.write('[m] %s <%s>\n' % (name.encode('utf-8'), email))
+
+            maintainer.save()
+
+        return maintainer

euscanwww/euscan/management/commands/scan-portage.py

@@ -0,0 +1,147 @@
+import subprocess
+import portage
+import sys
+import os
+import re
+
+from portage import versions
+from optparse import make_option
+
+from django.db.transaction import commit_on_success
+from django.core.management.base import BaseCommand, CommandError
+from euscanwww.euscan.models import Package, Version
+
+class Command(BaseCommand):
+    _overlays = {}
+
+    option_list = BaseCommand.option_list + (
+        make_option('--all',
+            action='store_true',
+            dest='all',
+            default=False,
+            help='Scan all packages'),
+        make_option('--quiet',
+            action='store_true',
+            dest='quiet',
+            default=False,
+            help='Be quiet'),
+        )
+    args = '<package package ...>'
+    help = 'Scans portage tree and fills database'
+
+    def handle(self, *args, **options):
+        if len(args) == 0 and options['all'] == False:
+            raise CommandError('You must specify a package or use --all')
+
+        if not options['quiet']:
+            self.stdout.write('Scanning portage tree...\n')
+
+        Version.objects.all().delete()
+        Package.objects.all().delete()
+
+        for package in args:
+            self.scan(options, package)
+
+        if len(args) == 0:
+            self.scan(options)
+
+        if not options['quiet']:
+            self.stdout.write('Done.\n')
+
+    def overlays(self):
+        if self._overlays:
+            return self._overlays
+
+        env = os.environ
+        env['OVERLAYS_LIST'] = 'all'
+        env['PRINT_COUNT_ALWAYS'] = 'never'
+
+        cmd = ['eix', '-!']
+
+        output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
+        output = output.strip().strip('\n').split('\n')
+
+        overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
+
+        self._overlays = {}
+
+        for line in output:
+            match = overlay_re.match(line)
+            if not match:
+                continue
+            self._overlays[match.group('key')] = match.group('name')
+
+        return self._overlays
+
+    @commit_on_success
+    def scan(self, options, query=None):
+        env = os.environ
+        env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
+
+	cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages', '-x']
+	if query:
+		cmd.extend(['--exact', query])
+
+        output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
+        output = output.strip().strip('\n')
+
+        if len(output) == 0:
+            if package:
+                sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % package))
+            return
+
+	output = output.split('\n')
+
+        line_re = re.compile(r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$')
+
+        package = None
+
+	for line in output:
+            match = line_re.match(line)
+
+            if not match:
+                continue
+
+            cpv = match.group('cpv')
+            slot = match.group('slot')
+            overlay = match.group('overlay')
+
+            if not package or not cpv.startswith(str(package)):
+                package = self.store_package(options, cpv)
+
+            self.store_version(options, package, cpv, slot, overlay)
+
+    def store_package(self, options, cpv):
+        cat, pkg, ver, rev = portage.catpkgsplit(cpv)
+
+        obj, created = Package.objects.get_or_create(category=cat, name=pkg)
+
+        if created:
+            if not options['quiet']:
+                sys.stdout.write('[p] %s/%s\n' % (cat, pkg))
+            obj.save()
+
+        # Delete previous versions to handle incremental scan correctly
+        Version.objects.filter(package=obj, packaged=True).delete()
+
+        return obj
+
+    def store_version(self, options, package, cpv, slot, overlay):
+        cat, pkg, ver, rev = portage.catpkgsplit(cpv)
+
+        overlays = self.overlays()
+
+        if overlay in overlays:
+            overlay = overlays[overlay]
+        else:
+            overlay = 'gentoo'
+
+        if not options['quiet']:
+            sys.stdout.write('[v] %s:%s [%s]\n' % (cpv, slot, overlay))
+
+        obj, created = Version.objects.get_or_create(package=package, slot=slot,
+                                                     revision=rev, version=ver,
+                                                     overlay=overlay)
+        obj.packaged = True
+        #obj.save()
+