euscanwww: some optimizations

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>
This commit is contained in:
Corentin Chary 2012-08-03 21:59:09 +02:00
parent 94f6b7541c
commit 0c97fcd42e
3 changed files with 83 additions and 30 deletions

View File

@ -1,7 +1,7 @@
import os import os
from gentoolkit.query import Query import portage
from gentoolkit.dbapi import PORTDB from gentoolkit.metadata import MetaData
import xml.etree.cElementTree as etree import xml.etree.cElementTree as etree
@ -9,7 +9,7 @@ from django.db.transaction import commit_on_success
from django.core.management.color import color_style from django.core.management.color import color_style
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from djeuscan.models import Package, Herd, Maintainer from djeuscan.models import Package, Version, Herd, Maintainer
from djeuscan.processing import FakeLogger from djeuscan.processing import FakeLogger
@ -18,7 +18,43 @@ class ScanMetadata(object):
self.style = color_style() self.style = color_style()
self.logger = logger or FakeLogger() self.logger = logger or FakeLogger()
def scan(self, query=None, obj=None): def get_package(self, query):
try:
return Package.objects.get(name=query)
except Package.DoesNotExist:
pass
try:
category, package = portage.catsplit(query)
return Package.objects.get(category=package, name=package)
except Package.DoesNotExist:
pass
try:
category, package, ver, rev = portage.catpkgsplit(query)
return Package.objects.get(category=category, name=package)
except Package.DoesNotExist:
pass
return None
def metadata_from_db(self, query, pkg=None):
if not pkg:
pkg = self.get_package(query)
try:
version = Version.objects.filter(package=pkg).values('metadata_path')\
.order_by('version', 'revision')[0]
except IndexError:
return pkg, None
if not version['metadata_path']:
return pkg, None
return pkg, MetaData(version['metadata_path'])
def metadata_from_portage(self, query, pkg=None):
from gentoolkit.query import Query
matches = Query(query).smart_find( matches = Query(query).smart_find(
in_installed=True, in_installed=True,
in_porttree=True, in_porttree=True,
@ -32,16 +68,16 @@ class ScanMetadata(object):
self.logger.error( self.logger.error(
self.style.ERROR("Unknown package '%s'" % query) self.style.ERROR("Unknown package '%s'" % query)
) )
return return pkg, None
matches = sorted(matches) matches = sorted(matches)
pkg = matches.pop() package = matches.pop()
if '9999' in pkg.version and len(matches): if '9999' in package.version and len(matches):
pkg = matches.pop() package = matches.pop()
if not obj: if not pkg:
obj, created = Package.objects.get_or_create( pkg, created = Package.objects.get_or_create(
category=pkg.category, name=pkg.name category=package.category, name=package.name
) )
else: else:
created = False created = False
@ -49,44 +85,57 @@ class ScanMetadata(object):
if created: if created:
self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name)) self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name))
return pkg, package.metadata
def scan(self, query=None, pkg=None):
try: try:
if not pkg.metadata: metadata = None
pkg, metadata = self.metadata_from_db(query, pkg)
if not metadata:
pkg, metadata = self.metadata_from_portage(query, pkg)
if not metadata:
return return
except Exception as e: except Exception as e:
if pkg:
self.logger.error( self.logger.error(
self.style.ERROR('%s/%s: %s' % self.style.ERROR('%s/%s: %s' %
(pkg.category, pkg.name, str(e))) (pkg.category, pkg.name, str(e)))
) )
else:
self.logger.error(
self.style.ERROR('%s: %s' % (query, str(e)))
)
return return
herds = dict( herds = dict(
[(herd[0], herd) for herd in pkg.metadata.herds(True)] [(herd[0], herd) for herd in metadata.herds(True)]
) )
maintainers = dict( maintainers = dict(
[(m.email, m) for m in pkg.metadata.maintainers()] [(m.email, m) for m in metadata.maintainers()]
) )
existing_herds = [h.herd for h in obj.herds.all()] existing_herds = [h.herd for h in pkg.herds.all()]
new_herds = set(herds.keys()).difference(existing_herds) new_herds = set(herds.keys()).difference(existing_herds)
old_herds = set(existing_herds).difference(herds.keys()) old_herds = set(existing_herds).difference(herds.keys())
existing_maintainers = [m.email for m in obj.maintainers.all()] existing_maintainers = [m.email for m in pkg.maintainers.all()]
new_maintainers = set(maintainers.keys()).\ new_maintainers = set(maintainers.keys()).\
difference(existing_maintainers) difference(existing_maintainers)
old_maintainers = set(existing_maintainers).\ old_maintainers = set(existing_maintainers).\
difference(maintainers.keys()) difference(maintainers.keys())
for herd in obj.herds.all(): for herd in pkg.herds.all():
if herd.herd in old_herds: if herd.herd in old_herds:
obj.herds.remove(herd) pkg.herds.remove(herd)
for herd in new_herds: for herd in new_herds:
herd = self.store_herd(*herds[herd]) herd = self.store_herd(*herds[herd])
obj.herds.add(herd) pkg.herds.add(herd)
for maintainer in obj.maintainers.all(): for maintainer in pkg.maintainers.all():
if maintainer.email in old_maintainers: if maintainer.email in old_maintainers:
obj.maintainers.remove(maintainer) pkg.maintainers.remove(maintainer)
for maintainer in new_maintainers: for maintainer in new_maintainers:
maintainer = maintainers[maintainer] maintainer = maintainers[maintainer]
@ -94,13 +143,13 @@ class ScanMetadata(object):
maintainer = self.store_maintainer( maintainer = self.store_maintainer(
maintainer.name, maintainer.email maintainer.name, maintainer.email
) )
obj.maintainers.add(maintainer) pkg.maintainers.add(maintainer)
except ValidationError: except ValidationError:
self.logger.error( self.logger.error(
self.style.ERROR("Bad maintainer: '%s' '%s'" % \ self.style.ERROR("Bad maintainer: '%s' '%s'" % \
(maintainer.name, maintainer.email)) (maintainer.name, maintainer.email))
) )
obj.save() pkg.save()
def store_herd(self, name, email): def store_herd(self, name, email):
if not name: if not name:
@ -141,7 +190,7 @@ class ScanMetadata(object):
self.logger.info("Populating herds and maintainers from herds.xml...") self.logger.info("Populating herds and maintainers from herds.xml...")
herds_xml_path = herds_xml_path or os.path.join( herds_xml_path = herds_xml_path or os.path.join(
PORTDB.settings["PORTDIR"], "metadata", "herds.xml" portage.settings["PORTDIR"], "metadata", "herds.xml"
) )
try: try:
self._herdstree = etree.parse(herds_xml_path) self._herdstree = etree.parse(herds_xml_path)

View File

@ -1,4 +1,6 @@
import subprocess import subprocess
import os
from os.path import join from os.path import join
import portage import portage
@ -95,12 +97,14 @@ class ScanPortage(object):
def scan_eix_xml(self, query, category=None): def scan_eix_xml(self, query, category=None):
cmd = ['eix', '--xml'] cmd = ['eix', '--xml']
env = os.environ
env['XML_OVERLAY'] = 'true'
if query: if query:
cmd.extend(['--exact', query]) cmd.extend(['--exact', query])
if category: if category:
cmd.extend(['-C', category]) cmd.extend(['-C', category])
sub = subprocess.Popen(cmd, stdout=subprocess.PIPE) sub = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE)
output = sub.stdout output = sub.stdout
try: try: