2011-04-13 19:00:31 +02:00
|
|
|
import subprocess
|
|
|
|
import portage
|
|
|
|
import sys
|
|
|
|
import os
|
|
|
|
import re
|
|
|
|
|
|
|
|
from portage import versions
|
|
|
|
from optparse import make_option
|
|
|
|
|
|
|
|
from django.db.transaction import commit_on_success
|
|
|
|
from django.core.management.base import BaseCommand, CommandError
|
2011-08-25 15:39:54 +02:00
|
|
|
from euscanwww.euscan.models import Package, Version, VersionLog
|
2011-04-13 19:00:31 +02:00
|
|
|
|
|
|
|
class Command(BaseCommand):
|
|
|
|
_overlays = {}
|
|
|
|
|
|
|
|
option_list = BaseCommand.option_list + (
|
|
|
|
make_option('--all',
|
|
|
|
action='store_true',
|
|
|
|
dest='all',
|
|
|
|
default=False,
|
|
|
|
help='Scan all packages'),
|
2011-08-25 15:39:54 +02:00
|
|
|
make_option('--purge-packages',
|
2011-04-29 09:24:47 +02:00
|
|
|
action='store_true',
|
2011-08-25 15:39:54 +02:00
|
|
|
dest='purge-packages',
|
2011-04-29 09:24:47 +02:00
|
|
|
default=False,
|
|
|
|
help='Purge old packages'),
|
2011-08-25 15:39:54 +02:00
|
|
|
make_option('--purge-versions',
|
|
|
|
action='store_true',
|
|
|
|
dest='purge-versions',
|
|
|
|
default=False,
|
|
|
|
help='Purge old versions'),
|
2012-03-14 14:07:06 +01:00
|
|
|
make_option('--no-log',
|
|
|
|
action='store_true',
|
|
|
|
dest='no-log',
|
|
|
|
default=False,
|
|
|
|
help='Don\'t store logs'),
|
|
|
|
make_option('--prefetch',
|
|
|
|
action='store_true',
|
|
|
|
dest='prefetch',
|
|
|
|
default=False,
|
|
|
|
help='Prefetch all versions and packages from DB to speedup full scan process.'),
|
2011-04-13 19:00:31 +02:00
|
|
|
make_option('--quiet',
|
|
|
|
action='store_true',
|
|
|
|
dest='quiet',
|
|
|
|
default=False,
|
|
|
|
help='Be quiet'),
|
|
|
|
)
|
2012-03-02 14:08:28 +01:00
|
|
|
args = '[package package ...]'
|
2011-04-13 19:00:31 +02:00
|
|
|
help = 'Scans portage tree and fills database'
|
|
|
|
|
2012-03-14 14:07:06 +01:00
|
|
|
_cache = {'packages' : {}, 'versions' : {}}
|
|
|
|
|
|
|
|
def cache_hash_package(self, category, name):
|
|
|
|
return '%s/%s' % (category, name)
|
|
|
|
|
|
|
|
def cache_store_package(self, package):
|
|
|
|
key = self.cache_hash_package(package.category, package.name)
|
|
|
|
self._cache['packages'][key] = package
|
|
|
|
|
|
|
|
def cache_get_package(self, category, name):
|
|
|
|
return self._cache['packages'].get(self.cache_hash_package(category, name))
|
|
|
|
|
|
|
|
def cache_hash_version(self, category, name, version, revision, slot, overlay):
|
|
|
|
key = '%s/%s-%s-r%s %s %s' % (category, name,
|
|
|
|
version, revision,
|
|
|
|
slot, overlay)
|
|
|
|
return key
|
|
|
|
|
|
|
|
def cache_get_version(self, category, name, version, revision, slot, overlay):
|
|
|
|
key = self.cache_hash_version(category, name, version, revision, slot, overlay)
|
|
|
|
return self._cache['versions'].get(key)
|
|
|
|
|
|
|
|
def cache_store_version(self, version):
|
|
|
|
key = self.cache_hash_version(version.package.category, version.package.name,
|
|
|
|
version.version, version.revision, version.slot,
|
|
|
|
version.overlay)
|
|
|
|
self._cache['versions'][key] = version
|
|
|
|
|
2011-04-13 19:00:31 +02:00
|
|
|
def handle(self, *args, **options):
|
2012-03-14 14:07:06 +01:00
|
|
|
self.options = options
|
|
|
|
|
2011-04-13 19:00:31 +02:00
|
|
|
if not options['quiet']:
|
|
|
|
self.stdout.write('Scanning portage tree...\n')
|
|
|
|
|
2012-03-14 14:07:06 +01:00
|
|
|
if options['prefetch']:
|
|
|
|
if not options['quiet']:
|
|
|
|
self.stdout.write('Prefetching objects...')
|
|
|
|
self.stdout.flush()
|
|
|
|
for package in Package.objects.all():
|
|
|
|
self.cache_store_package(package)
|
|
|
|
for version in Version.objects.select_related('package').all():
|
|
|
|
self.cache_store_version(version)
|
|
|
|
if not options['quiet']:
|
|
|
|
self.stdout.write('done\n')
|
|
|
|
|
2012-03-02 14:08:28 +01:00
|
|
|
if options['all']:
|
2012-03-14 14:07:06 +01:00
|
|
|
self.scan()
|
2012-03-02 14:08:28 +01:00
|
|
|
elif len(args):
|
2011-04-29 09:24:47 +02:00
|
|
|
for package in args:
|
2012-03-14 14:07:06 +01:00
|
|
|
self.scan(package)
|
2012-03-02 14:08:28 +01:00
|
|
|
else:
|
|
|
|
for package in sys.stdin.readlines():
|
2012-03-14 14:07:06 +01:00
|
|
|
self.scan(package[:-1])
|
2011-04-13 19:00:31 +02:00
|
|
|
|
2011-08-25 15:39:54 +02:00
|
|
|
if options['purge-versions']:
|
|
|
|
self.purge_versions(options)
|
|
|
|
|
2011-04-13 19:00:31 +02:00
|
|
|
if not options['quiet']:
|
|
|
|
self.stdout.write('Done.\n')
|
|
|
|
|
|
|
|
def overlays(self):
|
|
|
|
if self._overlays:
|
|
|
|
return self._overlays
|
|
|
|
|
|
|
|
env = os.environ
|
|
|
|
env['OVERLAYS_LIST'] = 'all'
|
|
|
|
env['PRINT_COUNT_ALWAYS'] = 'never'
|
|
|
|
|
|
|
|
cmd = ['eix', '-!']
|
|
|
|
|
|
|
|
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
|
|
|
|
output = output.strip().strip('\n').split('\n')
|
|
|
|
|
|
|
|
overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
|
|
|
|
|
|
|
|
self._overlays = {}
|
|
|
|
|
|
|
|
for line in output:
|
|
|
|
match = overlay_re.match(line)
|
|
|
|
if not match:
|
|
|
|
continue
|
|
|
|
self._overlays[match.group('key')] = match.group('name')
|
|
|
|
|
|
|
|
return self._overlays
|
|
|
|
|
|
|
|
@commit_on_success
|
2012-03-14 14:07:06 +01:00
|
|
|
def scan(self, query=None):
|
2011-04-13 19:00:31 +02:00
|
|
|
env = os.environ
|
|
|
|
env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
|
|
|
|
|
|
|
|
cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages', '-x']
|
|
|
|
if query:
|
|
|
|
cmd.extend(['--exact', query])
|
|
|
|
|
2012-03-14 14:07:06 +01:00
|
|
|
if self.options['all']:
|
|
|
|
if not self.options['quiet']:
|
|
|
|
self.stdout.write('Killing existing versions...')
|
|
|
|
self.stdout.flush()
|
|
|
|
Version.objects.filter(packaged=True).update(alive=False)
|
|
|
|
if not self.options['quiet']:
|
|
|
|
self.stdout.write('done\n')
|
|
|
|
|
2011-04-13 19:00:31 +02:00
|
|
|
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
|
|
|
|
output = output.strip().strip('\n')
|
|
|
|
|
|
|
|
if len(output) == 0:
|
2011-04-29 09:24:47 +02:00
|
|
|
if not query:
|
|
|
|
return
|
2012-03-14 14:07:06 +01:00
|
|
|
if self.options['purge-packages']:
|
|
|
|
if not self.options['quiet']:
|
2011-08-25 15:39:54 +02:00
|
|
|
sys.stdout.write('- [p] %s\n' % (query))
|
2011-04-29 09:24:47 +02:00
|
|
|
if '/' in query:
|
|
|
|
cat, pkg = portage.catsplit(query)
|
|
|
|
Package.objects.filter(category=cat, name=pkg).delete()
|
|
|
|
else:
|
|
|
|
Package.objects.filter(name=query).delete()
|
|
|
|
else:
|
|
|
|
sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % query))
|
2011-04-13 19:00:31 +02:00
|
|
|
return
|
|
|
|
|
|
|
|
output = output.split('\n')
|
2011-04-29 09:24:47 +02:00
|
|
|
packages = {}
|
2011-04-13 19:00:31 +02:00
|
|
|
|
|
|
|
line_re = re.compile(r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$')
|
|
|
|
|
|
|
|
package = None
|
|
|
|
|
|
|
|
for line in output:
|
|
|
|
match = line_re.match(line)
|
|
|
|
|
|
|
|
if not match:
|
|
|
|
continue
|
|
|
|
|
|
|
|
cpv = match.group('cpv')
|
|
|
|
slot = match.group('slot')
|
|
|
|
overlay = match.group('overlay')
|
|
|
|
|
2011-04-25 22:27:32 +02:00
|
|
|
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
|
2011-04-13 19:00:31 +02:00
|
|
|
|
2011-04-29 09:24:47 +02:00
|
|
|
packages['%s/%s' % (cat, pkg)] = True
|
2011-05-05 18:10:47 +02:00
|
|
|
|
2011-04-25 22:27:32 +02:00
|
|
|
if not package or not (cat == package.category and pkg == package.name):
|
2012-03-14 14:07:06 +01:00
|
|
|
package = self.store_package(cat, pkg)
|
2011-04-13 19:00:31 +02:00
|
|
|
|
2012-03-14 14:07:06 +01:00
|
|
|
self.store_version(package, cpv, slot, overlay)
|
2011-04-13 19:00:31 +02:00
|
|
|
|
2012-03-14 14:07:06 +01:00
|
|
|
if self.options['purge-packages'] and not query:
|
2011-04-29 09:24:47 +02:00
|
|
|
for package in Package.objects.all():
|
|
|
|
cp = "%s/%s" % (package.category, package.name)
|
|
|
|
if cp not in packages:
|
2012-03-14 14:07:06 +01:00
|
|
|
if not self.options['quiet']:
|
2011-08-25 15:39:54 +02:00
|
|
|
sys.stdout.write('- [p] %s\n' % (package))
|
2011-04-29 09:24:47 +02:00
|
|
|
package.delete()
|
|
|
|
|
2012-03-14 14:07:06 +01:00
|
|
|
def store_package(self, cat, pkg):
|
|
|
|
created = False
|
|
|
|
obj = self.cache_get_package(cat, pkg)
|
|
|
|
|
|
|
|
if not obj:
|
|
|
|
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
|
|
|
|
self.cache_store_package(obj)
|
2011-04-13 19:00:31 +02:00
|
|
|
|
|
|
|
if created:
|
2012-03-14 14:07:06 +01:00
|
|
|
if not self.options['quiet']:
|
2011-08-25 15:39:54 +02:00
|
|
|
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
|
2011-04-13 19:00:31 +02:00
|
|
|
|
2011-08-25 15:39:54 +02:00
|
|
|
' Set all versions dead, then set found versions alive and delete old versions '
|
2012-03-14 14:07:06 +01:00
|
|
|
if not self.options['all']:
|
|
|
|
Version.objects.filter(package=obj, packaged=True).update(alive=False)
|
2011-04-13 19:00:31 +02:00
|
|
|
|
|
|
|
return obj
|
|
|
|
|
2012-03-14 14:07:06 +01:00
|
|
|
def store_version(self, package, cpv, slot, overlay):
|
2011-04-13 19:00:31 +02:00
|
|
|
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
|
|
|
|
|
|
|
|
overlays = self.overlays()
|
|
|
|
|
|
|
|
if overlay in overlays:
|
|
|
|
overlay = overlays[overlay]
|
|
|
|
else:
|
|
|
|
overlay = 'gentoo'
|
|
|
|
|
2012-03-14 14:07:06 +01:00
|
|
|
created = False
|
|
|
|
obj = self.cache_get_version(package.category, package.name, ver, rev, slot, overlay)
|
|
|
|
if not obj:
|
|
|
|
obj, created = Version.objects.get_or_create(package=package, slot=slot,
|
|
|
|
revision=rev, version=ver,
|
|
|
|
overlay=overlay)
|
|
|
|
|
2011-08-25 15:39:54 +02:00
|
|
|
obj.alive = True
|
|
|
|
obj.packaged = True
|
|
|
|
obj.save()
|
2011-04-14 19:28:38 +02:00
|
|
|
|
2012-03-14 14:07:06 +01:00
|
|
|
if created:
|
|
|
|
self.cache_store_version(obj)
|
|
|
|
|
2011-08-25 15:39:54 +02:00
|
|
|
''' nothing to do (note: it can't be an upstream version because overlay can't be empty here) '''
|
|
|
|
if not created:
|
|
|
|
return
|
2011-04-14 19:28:38 +02:00
|
|
|
|
2012-03-14 14:07:06 +01:00
|
|
|
if not self.options['quiet']:
|
2011-08-25 15:39:54 +02:00
|
|
|
sys.stdout.write('+ [v] %s \n' % (obj))
|
|
|
|
|
|
|
|
if overlay == 'gentoo':
|
|
|
|
package.n_packaged += 1
|
|
|
|
else:
|
|
|
|
package.n_overlay += 1
|
|
|
|
package.n_versions += 1
|
2011-04-14 19:28:38 +02:00
|
|
|
package.save()
|
|
|
|
|
2012-03-14 14:07:06 +01:00
|
|
|
if self.options['no-log']:
|
|
|
|
return
|
|
|
|
|
|
|
|
entry = VersionLog.objects.create(package=obj.package,
|
|
|
|
action=VersionLog.VERSION_ADDED)
|
2011-08-25 15:39:54 +02:00
|
|
|
entry.slot = obj.slot
|
|
|
|
entry.revision = obj.revision
|
|
|
|
entry.version = obj.version
|
|
|
|
entry.overlay = obj.overlay
|
|
|
|
entry.save()
|
|
|
|
|
|
|
|
@commit_on_success
|
|
|
|
def purge_versions(self, options):
|
|
|
|
' For each dead versions '
|
|
|
|
for version in Version.objects.filter(packaged=True, alive=False):
|
|
|
|
if version.overlay == 'gentoo':
|
|
|
|
version.package.n_packaged -= 1
|
|
|
|
else:
|
|
|
|
version.package.n_overlay -= 1
|
|
|
|
version.package.n_versions -= 1
|
|
|
|
version.package.save()
|
|
|
|
|
2012-03-14 14:07:06 +01:00
|
|
|
if not self.options['quiet']:
|
2011-08-25 15:39:54 +02:00
|
|
|
sys.stdout.write('- [v] %s\n' % (version))
|
2012-03-14 14:07:06 +01:00
|
|
|
|
|
|
|
if self.options['no-log']:
|
|
|
|
continue
|
|
|
|
|
|
|
|
entry = VersionLog.objects.create(package=version.package,
|
|
|
|
action=VersionLog.VERSION_REMOVED)
|
|
|
|
entry.slot = version.slot
|
|
|
|
entry.revision = version.revision
|
|
|
|
entry.version = version.version
|
|
|
|
entry.overlay = version.overlay
|
|
|
|
entry.save()
|
|
|
|
|
2011-09-06 11:31:27 +02:00
|
|
|
Version.objects.filter(packaged=True, alive=False).delete()
|
2011-04-13 19:00:31 +02:00
|
|
|
|