import subprocess import portage import sys import os import re from datetime import datetime from portage import versions from optparse import make_option from django.db.transaction import commit_on_success from django.core.management.base import BaseCommand, CommandError from euscanwww.euscan.models import Package, Version, EuscanResult class Command(BaseCommand): _overlays = {} option_list = BaseCommand.option_list + ( make_option('--all', action='store_true', dest='all', default=False, help='Scan all packages'), make_option('--parallel', action='store_true', dest='parallel', default=False, help='Use GNU Parallel'), make_option('--quiet', action='store_true', dest='quiet', default=False, help='Be quiet'), ) args = '' help = 'Scans metadata and fills database' def handle(self, *args, **options): if len(args) == 0 and options['all'] == False: raise CommandError('You must specify a package or use --all') if not options['quiet']: self.stdout.write('Scanning upstream...\n') packages = [] if len(args) == 0: for pkg in Package.objects.all(): packages.append('%s/%s' % (pkg.category, pkg.name)) else: packages = list(args) self.scan(options, packages) if not options['quiet']: self.stdout.write('Done.\n') @commit_on_success def scan(self, options, packages=None): if options['parallel']: jobs = '\n'.join(packages) cmd = ['gparallel', '--jobs', '150%', 'euscan'] fp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE) output = fp.communicate(jobs)[0] self.parse_output(options, output) else: for package in packages: cmd = ['euscan', package] fp = subprocess.Popen(cmd, stdout=subprocess.PIPE) output = fp.communicate()[0] self.parse_output(options, output) def parse_output(self, options, output): from portage.versions import _cp package_re = re.compile(r'^ \* (?P' + _cp + ') \[(?P.*?)\]$') version_re = re.compile(r'^Upstream Version: (?P.*?) (?P.*?)$') package = None log = "" for line in output.split('\n'): match = package_re.match(line) if match: if package: self.store_result(options, package, log) cpv = match.group('cpv') package = self.store_package(options, cpv) log = line continue log += line match = version_re.match(line) if match: ver = match.group('ver') url = match.group('url') self.store_version(options, package, ver, url) if package: self.store_result(options, package, log) def store_result(self, options, package, log): obj = EuscanResult() obj.package = package obj.result = log obj.datetime = datetime.now() obj.save() def store_package(self, options, cpv): cat, pkg, ver, rev = portage.catpkgsplit(cpv) obj, created = Package.objects.get_or_create(category=cat, name=pkg) if created: if not options['quiet']: sys.stdout.write('[p] %s/%s\n' % (cat, pkg)) # Delete previous versions to handle incremental scan correctly Version.objects.filter(package=obj, packaged=False).delete() obj.n_versions = Version.objects.filter(package=obj).count() obj.save() return obj def store_version(self, options, package, ver, url): obj, created = Version.objects.get_or_create(package=package, slot='', revision='r0', version=ver, overlay='') if created or not obj.packaged: if not options['quiet']: sys.stdout.write('[u] %s/%s-%s %s\n' % (package.category, package.name, ver, url)) obj.url = url obj.packaged = False obj.save() package.n_versions += 1 package.save()