import subprocess import os from os.path import join import sys import portage from gentoolkit.metadata import MetaData from layman import Layman from xml.etree.ElementTree import iterparse, ParseError from django.db.transaction import commit_on_success from django.db import models from django.core.management.color import color_style from django.conf import settings from euscan.version import get_version_type from djeuscan.processing import FakeLogger from djeuscan.models import Package, Version, VersionLog, Category, Overlay class ScanPortage(object): def __init__(self, logger=None, no_log=False, purge_packages=False, purge_versions=False, upstream=False): self.logger = logger or FakeLogger() self.no_log = no_log self.purge_packages = purge_packages self.purge_versions = purge_versions self.upstream = upstream self.style = color_style() self._cache = {'packages': {}, 'versions': {}} self._overlays = None self._packages_updated = set() self._versions = set() self._versions_seen = set() def packages_updated(self): return list(self._packages_updated) def hash_package(self, category, name): return '%s/%s' % (category, name) def cache_store_package(self, package): key = self.hash_package(package.category, package.name) self._cache['packages'][key] = package def cache_get_package(self, category, name): return self._cache['packages'].get( self.hash_package(category, name) ) def hash_version(self, category, name, version, revision, overlay): key = '%s/%s-%s-r%s %s' % (category, name, version, revision, overlay) return key def cache_get_version(self, category, name, version, revision, overlay): key = self.hash_version(category, name, version, revision, overlay) return self._cache['versions'].get(key) def cache_store_version(self, version): key = self.hash_version( version.package.category, version.package.name, version.version, version.revision, version.overlay ) self._cache['versions'][key] = version self._versions.add(version) def scan_gentoopm(self, query, category=None): import gentoopm pm = gentoopm.get_package_manager() if category: packages = pm.stack.filter(key_category=category) elif query: packages = pm.stack.filter(query) else: packages = pm.stack package = {} package_name = None for p in packages: pkg = p.key.package if pkg != package_name: if package_name: yield package package_name = pkg package['package'] = p.key.package package['category'] = p.key.category package['homepage'] = ' '.join(p.homepages) package['description'] = p.description package['versions'] = [] package['versions'].append( (p._cpv, p.slot, p.repository or 'gentoo') ) if package_name: yield package def scan_eix_xml(self, query, category=None): cmd = ['eix', '--xml'] env = os.environ env['XML_OVERLAY'] = 'true' if query: cmd.extend(['--exact', query]) if category: cmd.extend(['-C', category]) sub = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE) output = sub.stdout try: parser = iterparse(output, ["start", "end"]) parser.next() # read root tag just for testing output except ParseError: if query: msg = "Unknown package '%s'" % query else: msg = "No packages." self.logger.error(self.style.ERROR(msg)) return package = {'versions': []} category = "" for event, elem in parser: if event == "start": # on tag opening if elem.tag == "category": category = elem.attrib["name"] elif elem.tag == "package": package["package"] = elem.attrib["name"] package["category"] = category elif elem.tag in ["description", "homepage"]: package[elem.tag] = elem.text or "" elif elem.tag == "version": # append version data to versions cpv = "%s/%s-%s" % ( package["category"], package["package"], elem.attrib["id"] ) slot = elem.attrib.get("slot", "0") overlay = elem.attrib.get("repository", "gentoo") overlay_path = elem.attrib.get("overlay", None) package["versions"].append( (cpv, slot, overlay, overlay_path) ) elif event == "end": # on tag closing if elem.tag == "package": # clean old data yield package package = {"versions": []} if elem.tag == "category": # clean old data category = "" elem.clear() def scan(self, query=None, category=None): for data in self.scan_eix_xml(query, category): #for data in self.scan_gentoopm(query, category): cat, pkg = data['category'], data['package'] package = self.store_package( cat, pkg, data['homepage'], data['description'] ) new_version = False for cpv, slot, overlay, overlay_path in data['versions']: obj, created = self.store_version( package, cpv, slot, overlay, overlay_path ) self._versions_seen.add(obj) new_version = created or new_version # If the package has at least one new version scan upstream for it if new_version: self._packages_updated.add(package) self.purge_old_versions() self.purge_old_packages() def store_package(self, cat, pkg, homepage, description): created = False obj = self.cache_get_package(cat, pkg) if not obj: obj, created = Package.objects.get_or_create( category=cat, name=pkg, defaults={"homepage": homepage, "description": description}, ) self.cache_store_package(obj) if created: self.logger.info('+ [p] %s/%s' % (cat, pkg)) return obj def store_version(self, package, cpv, slot, overlay, overlay_path): cat, pkg, ver, rev = portage.catpkgsplit(cpv) if not overlay: overlay = 'gentoo' created = False obj = self.cache_get_version( package.category, package.name, ver, rev, overlay ) overlay_path = overlay_path or portage.settings["PORTDIR"] package_path = join(overlay_path, package.category, package.name) ebuild_path = join(package_path, "%s.ebuild" % cpv.split("/")[-1]) metadata_path = join(package_path, "metadata.xml") if not obj: obj, created = Version.objects.get_or_create( package=package, revision=rev, version=ver, overlay=overlay, defaults={ "slot": slot, "packaged": True, "vtype": get_version_type(ver), "confidence": 100, "handler": "portage", "ebuild_path": ebuild_path, "metadata_path": metadata_path, } ) if not created: # Created objects have defaults values if obj.slot != slot or obj.package != True: obj.slot = slot obj.packaged = True obj.save() if created: self.cache_store_version(obj) # nothing to do (note: it can't be an upstream version because # overlay can't be empty here) if not created: return obj, created # New version created self.logger.info('+ [v] %s' % (obj)) if overlay == 'gentoo': package.n_packaged += 1 else: package.n_overlay += 1 package.n_versions += 1 package.save() if not self.no_log: VersionLog.objects.create( package=obj.package, action=VersionLog.VERSION_ADDED, slot=obj.slot, revision=obj.revision, version=obj.version, overlay=obj.overlay, vtype=obj.vtype, ) return obj, created def purge_old_packages(self): if not self.purge_packages: return packages = ( Package.objects.values("id") .annotate(version_count=models.Count("version")) .filter(version_count=0) ) packages = ( Package.objects.filter( id__in=[package['id'] for package in packages] ) ) for package in packages: self.logger.info('- [p] %s' % (package)) package.delete() def version_hack(self, version): try: if version.package.last_version_gentoo: version.package.last_version_gentoo.pk if version.package.last_version_overlay: version.package.last_version_overlay.pk if version.package.last_version_upstream: version.package.last_version_upstream.pk except Version.DoesNotExist: version.package.last_version_gentoo = None version.package.last_version_overlay = None version.package.last_version_upstream = None def purge_old_versions(self): if not self.purge_versions: return versions = self._versions.difference(self._versions_seen) for version in versions: self.logger.info('- [v] %s' % (version)) if version.packaged == False: continue # Not our job # Fix last_version_ stuff that is sometime broken self.version_hack(version) if version.overlay == 'gentoo': version.package.n_packaged -= 1 else: version.package.n_overlay -= 1 version.package.n_versions -= 1 version.package.save() if self.no_log: continue VersionLog.objects.create( package=version.package, action=VersionLog.VERSION_REMOVED, slot=version.slot, revision=version.revision, version=version.version, overlay=version.overlay, vtype=version.vtype, ) # remove from last version ? version.delete() def prefetch(self, packages, category): self.logger.info('Prefetching current objects...') ppackages = Package.objects.all() pversions = Version.objects.filter(packaged=True)\ .select_related('package').all() if category: ppackages = ppackages.filter(category=category) pversions = pversions.filter(package__category=category) if packages: ids = [package.id for package in packages] ppackages = ppackages.filter(pk__in=ids) pversions = pversions.filter(package__pk__in=ids) for package in ppackages: self.cache_store_package(package) for version in pversions: self.cache_store_version(version) self.logger.info('done') def populate_categories(logger): portdir = portage.settings["PORTDIR"] for cat in portage.settings.categories: try: meta = MetaData(join(portdir, cat, "metadata.xml")) desc = meta.descriptions()[0] except (IOError, IndexError): desc = "" obj, created = Category.objects.get_or_create(name=cat) obj.description = desc.strip() obj.save() if created: logger.info("+ [c] %s", cat) def populate_overlays(logger): l = Layman(stderr=sys.__stderr__, stdin=sys.__stdin__, stdout=sys.__stdout__, config=settings.LAYMAN_CONFIG, root="/") installed_overlays = l.get_installed() info = l.get_all_info(installed_overlays) for overlay in installed_overlays: if not overlay: continue obj, created = Overlay.objects.get_or_create(name=overlay) if overlay in info: obj.description = info[overlay]["description"] obj.homepage = info[overlay]["homepage"] obj.overlay_path = join(l.config['storage'], overlay) obj.save() if created: logger.info("+ [o] %s", overlay) @commit_on_success def scan_portage(packages=None, category=None, no_log=False, upstream=False, purge_packages=False, purge_versions=False, logger=None): logger = logger or FakeLogger() scan_handler = ScanPortage( logger=logger, no_log=no_log, purge_packages=purge_packages, purge_versions=purge_versions ) logger.info('Scanning portage tree...') if not packages: qs = Package.objects.all() if category: qs = qs.filter(category=category) prefetch_packages = qs else: results = [] for package in packages: if isinstance(package, Package): results.append(package) else: if '/' in package: cat, pkg = portage.catsplit(package) qs = Package.objects.filter(category=cat, name=pkg) else: qs = Package.objects.filter(name=package) for package in qs: results.append(package) prefetch_packages = results scan_handler.prefetch(prefetch_packages, category) if not packages and category: scan_handler.scan(category=category) elif not packages: scan_handler.scan() else: for pkg in packages: if isinstance(pkg, Package): scan_handler.scan('%s/%s' % (pkg.category, pkg.name)) else: scan_handler.scan(pkg) populate_categories(logger) populate_overlays(logger) logger.info('Done.') return scan_handler.packages_updated()