src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

djeuscan/scan_portage: fix purge-packages and purge-versions

Browse Source 
purge-packages: purge package that are not found in the tree
purge-versions: purge versions that are not found in the tree

Also, only purge versions of packages specified on the command line
when called without --all.

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>
This commit is contained in:
Corentin Chary 2012-06-30 10:19:07 +02:00
parent 20f796df60
commit 9e6f885f89
2 changed files with 83 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
euscanwww/djeuscan/management/commands/scan_portage.py
View File

@ -49,7 +49,6 @@ class Command(BaseCommand):
if options['all']: if options['all']:
packages = None packages = None
elif len(args): elif len(args):
packages = [pkg for pkg in args] packages = [pkg for pkg in args]
else: else:

121
euscanwww/djeuscan/processing/scan_portage.py
View File

@ -2,6 +2,7 @@ import subprocess
import portage import portage
import os import os
import re import re
from xml.dom.minidom import parseString from xml.dom.minidom import parseString
from django.db.transaction import commit_on_success from django.db.transaction import commit_on_success
@ -15,11 +16,11 @@ from djeuscan.models import Package, Version, VersionLog
class ScanPortage(object): class ScanPortage(object):
def __init__(self, logger=None, no_log=False, purge_packages=False, def __init__(self, logger=None, no_log=False, purge_packages=False,
kill_versions=False): purge_versions=False):
self.logger = logger or FakeLogger() self.logger = logger or FakeLogger()
self.no_log = no_log self.no_log = no_log
self.purge_packages = purge_packages self.purge_packages = purge_packages
self.kill_versions = kill_versions self.purge_versions = purge_versions
self.style = color_style() self.style = color_style()
self._cache = {'packages': {}, 'versions': {}} self._cache = {'packages': {}, 'versions': {}}
@ -57,62 +58,32 @@ class ScanPortage(object):
) )
self._cache['versions'][key] = version self._cache['versions'][key] = version
def overlays(self):
if self._overlays:
return self._overlays
env = os.environ
env['OVERLAYS_LIST'] = 'all'
env['PRINT_COUNT_ALWAYS'] = 'never'
cmd = ['eix', '-!']
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
output = output.strip().strip('\n').split('\n')
overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
self._overlays = {}
for line in output:
match = overlay_re.match(line)
if not match:
continue
self._overlays[match.group('key')] = match.group('name')
return self._overlays
def scan(self, query=None): def scan(self, query=None):
if self.purge_packages: cmd = ['eix', '--xml']
with commit_on_success():
for package in Package.objects.all():
self.logger.info('- [p] %s' % (package))
package.delete()
cmd = ['eix', '--xml', '--pure-packages', '-x']
if query: if query:
cmd.extend(['--exact', query]) cmd.extend(['--exact', query])
if self.kill_versions: if not query:
current_packages = Package.objects.all()
elif '/' in query:
cat, pkg = portage.catsplit(query)
current_packages = Package.objects.filter(category=cat, name=pkg)
else:
current_packages = Package.objects.filter(name=query)
if self.purge_versions:
if not query:
self.logger.info('Killing existing versions...') self.logger.info('Killing existing versions...')
Version.objects.filter(packaged=True).update(alive=False) Version.objects.filter(packaged=True).update(alive=False)
self.logger.info('done') self.logger.info('done')
else:
for package in current_packages:
Version.objects.filter(package=package, packaged=True).update(alive=False)
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).\ output = subprocess.Popen(cmd, stdout=subprocess.PIPE).\
communicate()[0] communicate()[0]
if len(output) == 0: if len(output) == 0 and query:
if not query:
return
if self.purge_packages:
self.logger.info('- [p] %s' % (query))
if '/' in query:
cat, pkg = portage.catsplit(query)
Package.objects.filter(category=cat, name=pkg).delete()
else:
Package.objects.filter(name=query).delete()
else:
self.logger.error( self.logger.error(
self.style.ERROR( self.style.ERROR(
"Unknown package '%s'" % query "Unknown package '%s'" % query
@ -120,8 +91,10 @@ class ScanPortage(object):
) )
return return
dom = parseString(output) packages_alive = set()
dom = parseString(output)
# SAX
for category_tag in dom.getElementsByTagName("category"): for category_tag in dom.getElementsByTagName("category"):
for package_tag in category_tag.getElementsByTagName("package"): for package_tag in category_tag.getElementsByTagName("package"):
cat = category_tag.getAttribute("name") cat = category_tag.getAttribute("name")
@ -137,17 +110,26 @@ class ScanPortage(object):
except (IndexError, AttributeError): except (IndexError, AttributeError):
desc = "" desc = ""
with commit_on_success():
package = self.store_package(cat, pkg, homepage, desc) package = self.store_package(cat, pkg, homepage, desc)
packages_alive.add('%s/%s' % (cat, pkg))
for version_tag in package_tag.\ for version_tag in package_tag.\
getElementsByTagName("version"): getElementsByTagName("version"):
cpv = "%s/%s-%s" % (cat, pkg, cpv = "%s/%s-%s" % (cat, pkg,
version_tag.getAttribute("id")) version_tag.getAttribute("id"))
slot = version_tag.getAttribute("slot") slot = version_tag.getAttribute("slot")
overlay = version_tag.getAttribute("overlay") overlay = version_tag.getAttribute("repository")
self.store_version(package, cpv, slot, overlay) self.store_version(package, cpv, slot, overlay)
if self.purge_packages:
for package in current_packages:
cp = "%s/%s" % (package.category, package.name)
if cp not in packages_alive:
self.logger.info('- [p] %s' % (package))
package.delete()
if self.purge_versions:
self.purge_old_versions(current_packages)
def store_package(self, cat, pkg, homepage, description): def store_package(self, cat, pkg, homepage, description):
created = False created = False
obj = self.cache_get_package(cat, pkg) obj = self.cache_get_package(cat, pkg)
@ -166,7 +148,7 @@ class ScanPortage(object):
# Set all versions dead, then set found versions alive and # Set all versions dead, then set found versions alive and
# delete old versions # delete old versions
if not self.kill_versions: if not self.purge_versions:
Version.objects.filter( Version.objects.filter(
package=obj, package=obj,
packaged=True packaged=True
@ -176,12 +158,7 @@ class ScanPortage(object):
def store_version(self, package, cpv, slot, overlay): def store_version(self, package, cpv, slot, overlay):
cat, pkg, ver, rev = portage.catpkgsplit(cpv) cat, pkg, ver, rev = portage.catpkgsplit(cpv)
if not overlay:
overlays = self.overlays()
if overlay in overlays:
overlay = overlays[overlay]
else:
overlay = 'gentoo' overlay = 'gentoo'
created = False created = False
@ -235,13 +212,18 @@ class ScanPortage(object):
overlay=obj.overlay overlay=obj.overlay
) )
def purge_old_versions(self, packages):
@commit_on_success
def do_purge_versions(logger=None, no_log=False):
logger = logger or FakeLogger()
# For each dead versions # For each dead versions
for version in Version.objects.filter(packaged=True, alive=False): if packages:
versions = []
for package in packages:
qs = Version.objects.filter(package=package, packaged=True, alive=False)
for version in qs:
versions.append(version)
else:
versions = Version.objects.filter(packaged=True, alive=False)
for version in versions:
if version.overlay == 'gentoo': if version.overlay == 'gentoo':
version.package.n_packaged -= 1 version.package.n_packaged -= 1
else: else:
@ -249,9 +231,9 @@ def do_purge_versions(logger=None, no_log=False):
version.package.n_versions -= 1 version.package.n_versions -= 1
version.package.save() version.package.save()
logger.info('- [v] %s' % (version)) self.logger.info('- [v] %s' % (version))
if no_log: if self.no_log:
continue continue
VersionLog.objects.create( VersionLog.objects.create(
@ -265,21 +247,20 @@ def do_purge_versions(logger=None, no_log=False):
Version.objects.filter(packaged=True, alive=False).delete() Version.objects.filter(packaged=True, alive=False).delete()
@commit_on_success
def scan_portage(packages=None, no_log=False, purge_packages=False, def scan_portage(packages=None, no_log=False, purge_packages=False,
purge_versions=False, prefetch=False, logger=None): purge_versions=False, prefetch=False, logger=None):
logger = logger or FakeLogger() logger = logger or FakeLogger()
kill_versions = False
if packages is None: if packages is None:
prefetch = True prefetch = True
kill_versions = True
scan_handler = ScanPortage( scan_handler = ScanPortage(
logger=logger, logger=logger,
no_log=no_log, no_log=no_log,
purge_packages=purge_packages, purge_packages=purge_packages,
kill_versions=kill_versions, purge_versions=purge_versions,
) )
logger.info('Scanning portage tree...') logger.info('Scanning portage tree...')
@ -301,7 +282,5 @@ def scan_portage(packages=None, no_log=False, purge_packages=False,
else: else:
scan_handler.scan(pkg) scan_handler.scan(pkg)
if purge_versions:
do_purge_versions(logger=logger, no_log=no_log)
logger.info('Done.') logger.info('Done.')
return True return True