Conflicts:
	euscanwww/djeuscan/processing/scan_portage.py

Signed-off-by: volpino <fox91@anche.no>
This commit is contained in:
volpino 2012-06-30 10:39:18 +02:00
commit c184628f48
2 changed files with 73 additions and 84 deletions

View File

@ -49,7 +49,6 @@ class Command(BaseCommand):
if options['all']: if options['all']:
packages = None packages = None
elif len(args): elif len(args):
packages = [pkg for pkg in args] packages = [pkg for pkg in args]
else: else:

View File

@ -1,7 +1,6 @@
import subprocess import subprocess
import portage import portage
import os
import re
from xml.etree.ElementTree import iterparse, ParseError from xml.etree.ElementTree import iterparse, ParseError
from django.db.transaction import commit_on_success from django.db.transaction import commit_on_success
@ -15,11 +14,11 @@ from djeuscan.models import Package, Version, VersionLog
class ScanPortage(object): class ScanPortage(object):
def __init__(self, logger=None, no_log=False, purge_packages=False, def __init__(self, logger=None, no_log=False, purge_packages=False,
kill_versions=False): purge_versions=False):
self.logger = logger or FakeLogger() self.logger = logger or FakeLogger()
self.no_log = no_log self.no_log = no_log
self.purge_packages = purge_packages self.purge_packages = purge_packages
self.kill_versions = kill_versions self.purge_versions = purge_versions
self.style = color_style() self.style = color_style()
self._cache = {'packages': {}, 'versions': {}} self._cache = {'packages': {}, 'versions': {}}
@ -57,47 +56,28 @@ class ScanPortage(object):
) )
self._cache['versions'][key] = version self._cache['versions'][key] = version
def overlays(self):
if self._overlays:
return self._overlays
env = os.environ
env['OVERLAYS_LIST'] = 'all'
env['PRINT_COUNT_ALWAYS'] = 'never'
cmd = ['eix', '-!']
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
output = output.strip().strip('\n').split('\n')
overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
self._overlays = {}
for line in output:
match = overlay_re.match(line)
if not match:
continue
self._overlays[match.group('key')] = match.group('name')
return self._overlays
def scan(self, query=None): def scan(self, query=None):
if self.purge_packages: cmd = ['eix', '--xml']
with commit_on_success():
for package in Package.objects.all():
self.logger.info('- [p] %s' % (package))
package.delete()
cmd = ['eix', '--xml', '--pure-packages', '-x']
if query: if query:
cmd.extend(['--exact', query]) cmd.extend(['--exact', query])
if self.kill_versions: if not query:
self.logger.info('Killing existing versions...') current_packages = Package.objects.all()
Version.objects.filter(packaged=True).update(alive=False) elif '/' in query:
self.logger.info('done') cat, pkg = portage.catsplit(query)
current_packages = Package.objects.filter(category=cat, name=pkg)
else:
current_packages = Package.objects.filter(name=query)
if self.purge_versions:
if not query:
self.logger.info('Killing existing versions...')
Version.objects.filter(packaged=True).update(alive=False)
self.logger.info('done')
else:
for package in current_packages:
Version.objects.filter(package=package, packaged=True).\
update(alive=False)
sub = subprocess.Popen(cmd, stdout=subprocess.PIPE) sub = subprocess.Popen(cmd, stdout=subprocess.PIPE)
@ -122,10 +102,11 @@ class ScanPortage(object):
"Unknown package '%s'" % query "Unknown package '%s'" % query
) )
) )
return return
cat, pkg, homepage, desc = ("", "", "", "") cat, pkg, homepage, desc = ("", "", "", "")
versions = [] versions = []
packages_alive = set()
for event, elem in parser: for event, elem in parser:
if event == "start": # on tag opening if event == "start": # on tag opening
@ -149,6 +130,7 @@ class ScanPortage(object):
# package tag has been closed, saving everything! # package tag has been closed, saving everything!
with commit_on_success(): with commit_on_success():
package = self.store_package(cat, pkg, homepage, desc) package = self.store_package(cat, pkg, homepage, desc)
packages_alive.add('%s/%s' % (cat, pkg))
for cpv, slot, overlay in versions: for cpv, slot, overlay in versions:
self.store_version(package, cpv, slot, overlay) self.store_version(package, cpv, slot, overlay)
@ -160,6 +142,15 @@ class ScanPortage(object):
# clean old data # clean old data
cat = "" cat = ""
if self.purge_packages:
for package in current_packages:
cp = "%s/%s" % (package.category, package.name)
if cp not in packages_alive:
self.logger.info('- [p] %s' % (package))
package.delete()
if self.purge_versions:
self.purge_old_versions(current_packages)
def store_package(self, cat, pkg, homepage, description): def store_package(self, cat, pkg, homepage, description):
created = False created = False
obj = self.cache_get_package(cat, pkg) obj = self.cache_get_package(cat, pkg)
@ -178,7 +169,7 @@ class ScanPortage(object):
# Set all versions dead, then set found versions alive and # Set all versions dead, then set found versions alive and
# delete old versions # delete old versions
if not self.kill_versions: if not self.purge_versions:
Version.objects.filter( Version.objects.filter(
package=obj, package=obj,
packaged=True packaged=True
@ -188,12 +179,7 @@ class ScanPortage(object):
def store_version(self, package, cpv, slot, overlay): def store_version(self, package, cpv, slot, overlay):
cat, pkg, ver, rev = portage.catpkgsplit(cpv) cat, pkg, ver, rev = portage.catpkgsplit(cpv)
if not overlay:
overlays = self.overlays()
if overlay in overlays:
overlay = overlays[overlay]
else:
overlay = 'gentoo' overlay = 'gentoo'
created = False created = False
@ -247,51 +233,57 @@ class ScanPortage(object):
overlay=obj.overlay overlay=obj.overlay
) )
def purge_old_versions(self, packages):
# For each dead versions
if packages:
versions = []
for package in packages:
qs = Version.objects.filter(package=package, packaged=True,
alive=False)
for version in qs:
versions.append(version)
else:
versions = Version.objects.filter(packaged=True, alive=False)
for version in versions:
if version.overlay == 'gentoo':
version.package.n_packaged -= 1
else:
version.package.n_overlay -= 1
version.package.n_versions -= 1
version.package.save()
self.logger.info('- [v] %s' % (version))
if self.no_log:
continue
VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED,
slot=version.slot,
revision=version.revision,
version=version.version,
overlay=version.overlay
)
Version.objects.filter(packaged=True, alive=False).delete()
@commit_on_success @commit_on_success
def do_purge_versions(logger=None, no_log=False):
logger = logger or FakeLogger()
# For each dead versions
for version in Version.objects.filter(packaged=True, alive=False):
if version.overlay == 'gentoo':
version.package.n_packaged -= 1
else:
version.package.n_overlay -= 1
version.package.n_versions -= 1
version.package.save()
logger.info('- [v] %s' % (version))
if no_log:
continue
VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED,
slot=version.slot,
revision=version.revision,
version=version.version,
overlay=version.overlay
)
Version.objects.filter(packaged=True, alive=False).delete()
def scan_portage(packages=None, no_log=False, purge_packages=False, def scan_portage(packages=None, no_log=False, purge_packages=False,
purge_versions=False, prefetch=False, logger=None): purge_versions=False, prefetch=False, logger=None):
logger = logger or FakeLogger() logger = logger or FakeLogger()
kill_versions = False
if packages is None: if packages is None:
prefetch = True prefetch = True
kill_versions = True
scan_handler = ScanPortage( scan_handler = ScanPortage(
logger=logger, logger=logger,
no_log=no_log, no_log=no_log,
purge_packages=purge_packages, purge_packages=purge_packages,
kill_versions=kill_versions, purge_versions=purge_versions,
) )
logger.info('Scanning portage tree...') logger.info('Scanning portage tree...')
@ -313,7 +305,5 @@ def scan_portage(packages=None, no_log=False, purge_packages=False,
else: else:
scan_handler.scan(pkg) scan_handler.scan(pkg)
if purge_versions:
do_purge_versions(logger=logger, no_log=no_log)
logger.info('Done.') logger.info('Done.')
return True return True