euscanwww: optimize commands

Signed-off-by: Corentin Chary <corentincj@iksaif.net>
This commit is contained in:
Corentin Chary 2012-03-14 14:07:06 +01:00
parent 043197a3be
commit 2effd34b01
3 changed files with 162 additions and 48 deletions

View File

@ -32,19 +32,21 @@ class Command(BaseCommand):
args = '<package package ...>' args = '<package package ...>'
help = 'Scans metadata and fills database' help = 'Scans metadata and fills database'
@commit_on_success
def handle(self, *args, **options): def handle(self, *args, **options):
self.options = options
if options['all']: if options['all']:
for pkg in Package.objects.all(): for pkg in Package.objects.all():
self.scan(options, '%s/%s' % (pkg.category, pkg.name)) self.scan('%s/%s' % (pkg.category, pkg.name), pkg)
elif len(args): elif len(args):
for package in args: for package in args:
self.scan(options, package) self.scan(package)
else: else:
for package in sys.stdin.readlines(): for package in sys.stdin.readlines():
self.scan(options, package[:-1]) self.scan(package[:-1])
@commit_on_success def scan(self, query=None, obj=None):
def scan(self, options, query=None):
matches = Query(query).find( matches = Query(query).find(
include_masked=True, include_masked=True,
in_installed=False, in_installed=False,
@ -59,7 +61,10 @@ class Command(BaseCommand):
if '9999' in pkg.version and len(matches): if '9999' in pkg.version and len(matches):
pkg = matches.pop() pkg = matches.pop()
if not obj:
obj, created = Package.objects.get_or_create(category=pkg.category, name=pkg.name) obj, created = Package.objects.get_or_create(category=pkg.category, name=pkg.name)
else:
created = False
try: try:
obj.homepage = pkg.environment("HOMEPAGE") obj.homepage = pkg.environment("HOMEPAGE")
@ -67,30 +72,48 @@ class Command(BaseCommand):
except GentoolkitFatalError, err: except GentoolkitFatalError, err:
sys.stderr.write(self.style.ERROR("Gentoolkit fatal error: '%s'\n" % str(err))) sys.stderr.write(self.style.ERROR("Gentoolkit fatal error: '%s'\n" % str(err)))
if created and not options['quiet']: if created and not self.options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name)) sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
if pkg.metadata:
obj.herds.clear()
obj.maintainers.clear()
for herd in pkg.metadata.herds(True): if pkg.metadata:
herd = self.store_herd(options, herd[0], herd[1]) herds = dict([(herd[0], herd) for herd in pkg.metadata.herds(True)])
maintainers = dict([(m.email, m) for m in pkg.metadata.maintainers()])
existing_herds = [h.herd for h in obj.herds.all()]
new_herds = set(herds.keys()).difference(existing_herds)
old_herds = set(existing_herds).difference(herds.keys())
existing_maintainers = [m.email for m in obj.maintainers.all()]
new_maintainers = set(maintainers.keys()).difference(existing_maintainers)
old_maintainers = set(existing_maintainers).difference(maintainers.keys())
for herd in obj.herds.all():
if herd.email in old_herds:
obj.herds.remove(herd)
for herd in new_herds:
herd = self.store_herd(*herds[herd])
obj.herds.add(herd) obj.herds.add(herd)
for maintainer in pkg.metadata.maintainers(): for maintainer in obj.maintainers.all():
maintainer = self.store_maintainer(options, maintainer.name, maintainer.email) if maintainer.email in old_maintainers:
obj.maintainers.remove(maintainer)
for maintainer in new_maintainers:
maintainer = maintainers[maintainer]
maintainer = self.store_maintainer(maintainer.name, maintainer.email)
obj.maintainers.add(maintainer) obj.maintainers.add(maintainer)
obj.save() obj.save()
def store_herd(self, options, name, email): def store_herd(self, name, email):
if not name: if not name:
name = '{nil}' name = '{nil}'
name = name.strip("\r").strip("\n").strip("\t").strip() name = name.strip("\r").strip("\n").strip("\t").strip()
herd, created = Herd.objects.get_or_create(herd=name) herd, created = Herd.objects.get_or_create(herd=name)
if created and not options['quiet']: if created and not self.options['quiet']:
sys.stdout.write('+ [h] %s <%s>\n' % (name, email)) sys.stdout.write('+ [h] %s <%s>\n' % (name, email))
herd.email = email herd.email = email
@ -98,7 +121,7 @@ class Command(BaseCommand):
return herd return herd
def store_maintainer(self, options, name, email): def store_maintainer(self, name, email):
if not name: if not name:
name = email name = email
if not name: if not name:
@ -107,9 +130,10 @@ class Command(BaseCommand):
maintainer, created = Maintainer.objects.get_or_create(email=email) maintainer, created = Maintainer.objects.get_or_create(email=email)
if created: if created:
if not options['quiet']: if not self.options['quiet']:
sys.stdout.write('+ [m] %s <%s>\n' % (name.encode('utf-8'), email)) sys.stdout.write('+ [m] %s <%s>\n' % (name.encode('utf-8'), email))
if not maintainer.name or name not in [maintainer.name, email, '{nil}']:
maintainer.name = name maintainer.name = name
maintainer.save() maintainer.save()

View File

@ -30,6 +30,16 @@ class Command(BaseCommand):
dest='purge-versions', dest='purge-versions',
default=False, default=False,
help='Purge old versions'), help='Purge old versions'),
make_option('--no-log',
action='store_true',
dest='no-log',
default=False,
help='Don\'t store logs'),
make_option('--prefetch',
action='store_true',
dest='prefetch',
default=False,
help='Prefetch all versions and packages from DB to speedup full scan process.'),
make_option('--quiet', make_option('--quiet',
action='store_true', action='store_true',
dest='quiet', dest='quiet',
@ -39,18 +49,59 @@ class Command(BaseCommand):
args = '[package package ...]' args = '[package package ...]'
help = 'Scans portage tree and fills database' help = 'Scans portage tree and fills database'
_cache = {'packages' : {}, 'versions' : {}}
def cache_hash_package(self, category, name):
return '%s/%s' % (category, name)
def cache_store_package(self, package):
key = self.cache_hash_package(package.category, package.name)
self._cache['packages'][key] = package
def cache_get_package(self, category, name):
return self._cache['packages'].get(self.cache_hash_package(category, name))
def cache_hash_version(self, category, name, version, revision, slot, overlay):
key = '%s/%s-%s-r%s %s %s' % (category, name,
version, revision,
slot, overlay)
return key
def cache_get_version(self, category, name, version, revision, slot, overlay):
key = self.cache_hash_version(category, name, version, revision, slot, overlay)
return self._cache['versions'].get(key)
def cache_store_version(self, version):
key = self.cache_hash_version(version.package.category, version.package.name,
version.version, version.revision, version.slot,
version.overlay)
self._cache['versions'][key] = version
def handle(self, *args, **options): def handle(self, *args, **options):
self.options = options
if not options['quiet']: if not options['quiet']:
self.stdout.write('Scanning portage tree...\n') self.stdout.write('Scanning portage tree...\n')
if options['prefetch']:
if not options['quiet']:
self.stdout.write('Prefetching objects...')
self.stdout.flush()
for package in Package.objects.all():
self.cache_store_package(package)
for version in Version.objects.select_related('package').all():
self.cache_store_version(version)
if not options['quiet']:
self.stdout.write('done\n')
if options['all']: if options['all']:
self.scan(options) self.scan()
elif len(args): elif len(args):
for package in args: for package in args:
self.scan(options, package) self.scan(package)
else: else:
for package in sys.stdin.readlines(): for package in sys.stdin.readlines():
self.scan(options, package[:-1]) self.scan(package[:-1])
if options['purge-versions']: if options['purge-versions']:
self.purge_versions(options) self.purge_versions(options)
@ -84,7 +135,7 @@ class Command(BaseCommand):
return self._overlays return self._overlays
@commit_on_success @commit_on_success
def scan(self, options, query=None): def scan(self, query=None):
env = os.environ env = os.environ
env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n" env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
@ -92,14 +143,22 @@ class Command(BaseCommand):
if query: if query:
cmd.extend(['--exact', query]) cmd.extend(['--exact', query])
if self.options['all']:
if not self.options['quiet']:
self.stdout.write('Killing existing versions...')
self.stdout.flush()
Version.objects.filter(packaged=True).update(alive=False)
if not self.options['quiet']:
self.stdout.write('done\n')
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0] output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
output = output.strip().strip('\n') output = output.strip().strip('\n')
if len(output) == 0: if len(output) == 0:
if not query: if not query:
return return
if options['purge-packages']: if self.options['purge-packages']:
if not options['quiet']: if not self.options['quiet']:
sys.stdout.write('- [p] %s\n' % (query)) sys.stdout.write('- [p] %s\n' % (query))
if '/' in query: if '/' in query:
cat, pkg = portage.catsplit(query) cat, pkg = portage.catsplit(query)
@ -132,31 +191,37 @@ class Command(BaseCommand):
packages['%s/%s' % (cat, pkg)] = True packages['%s/%s' % (cat, pkg)] = True
if not package or not (cat == package.category and pkg == package.name): if not package or not (cat == package.category and pkg == package.name):
package = self.store_package(options, cat, pkg) package = self.store_package(cat, pkg)
self.store_version(options, package, cpv, slot, overlay) self.store_version(package, cpv, slot, overlay)
if options['purge-packages'] and not query: if self.options['purge-packages'] and not query:
for package in Package.objects.all(): for package in Package.objects.all():
cp = "%s/%s" % (package.category, package.name) cp = "%s/%s" % (package.category, package.name)
if cp not in packages: if cp not in packages:
if not options['quiet']: if not self.options['quiet']:
sys.stdout.write('- [p] %s\n' % (package)) sys.stdout.write('- [p] %s\n' % (package))
package.delete() package.delete()
def store_package(self, options, cat, pkg): def store_package(self, cat, pkg):
created = False
obj = self.cache_get_package(cat, pkg)
if not obj:
obj, created = Package.objects.get_or_create(category=cat, name=pkg) obj, created = Package.objects.get_or_create(category=cat, name=pkg)
self.cache_store_package(obj)
if created: if created:
if not options['quiet']: if not self.options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg)) sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
' Set all versions dead, then set found versions alive and delete old versions ' ' Set all versions dead, then set found versions alive and delete old versions '
if not self.options['all']:
Version.objects.filter(package=obj, packaged=True).update(alive=False) Version.objects.filter(package=obj, packaged=True).update(alive=False)
return obj return obj
def store_version(self, options, package, cpv, slot, overlay): def store_version(self, package, cpv, slot, overlay):
cat, pkg, ver, rev = portage.catpkgsplit(cpv) cat, pkg, ver, rev = portage.catpkgsplit(cpv)
overlays = self.overlays() overlays = self.overlays()
@ -166,18 +231,25 @@ class Command(BaseCommand):
else: else:
overlay = 'gentoo' overlay = 'gentoo'
created = False
obj = self.cache_get_version(package.category, package.name, ver, rev, slot, overlay)
if not obj:
obj, created = Version.objects.get_or_create(package=package, slot=slot, obj, created = Version.objects.get_or_create(package=package, slot=slot,
revision=rev, version=ver, revision=rev, version=ver,
overlay=overlay) overlay=overlay)
obj.alive = True obj.alive = True
obj.packaged = True obj.packaged = True
obj.save() obj.save()
if created:
self.cache_store_version(obj)
''' nothing to do (note: it can't be an upstream version because overlay can't be empty here) ''' ''' nothing to do (note: it can't be an upstream version because overlay can't be empty here) '''
if not created: if not created:
return return
if not options['quiet']: if not self.options['quiet']:
sys.stdout.write('+ [v] %s \n' % (obj)) sys.stdout.write('+ [v] %s \n' % (obj))
if overlay == 'gentoo': if overlay == 'gentoo':
@ -187,7 +259,11 @@ class Command(BaseCommand):
package.n_versions += 1 package.n_versions += 1
package.save() package.save()
entry = VersionLog.objects.create(package=obj.package, action=VersionLog.VERSION_ADDED) if self.options['no-log']:
return
entry = VersionLog.objects.create(package=obj.package,
action=VersionLog.VERSION_ADDED)
entry.slot = obj.slot entry.slot = obj.slot
entry.revision = obj.revision entry.revision = obj.revision
entry.version = obj.version entry.version = obj.version
@ -198,13 +274,6 @@ class Command(BaseCommand):
def purge_versions(self, options): def purge_versions(self, options):
' For each dead versions ' ' For each dead versions '
for version in Version.objects.filter(packaged=True, alive=False): for version in Version.objects.filter(packaged=True, alive=False):
entry = VersionLog.objects.create(package=version.package, action=VersionLog.VERSION_REMOVED)
entry.slot = version.slot
entry.revision = version.revision
entry.version = version.version
entry.overlay = version.overlay
entry.save()
if version.overlay == 'gentoo': if version.overlay == 'gentoo':
version.package.n_packaged -= 1 version.package.n_packaged -= 1
else: else:
@ -212,7 +281,19 @@ class Command(BaseCommand):
version.package.n_versions -= 1 version.package.n_versions -= 1
version.package.save() version.package.save()
if not options['quiet']: if not self.options['quiet']:
sys.stdout.write('- [v] %s\n' % (version)) sys.stdout.write('- [v] %s\n' % (version))
if self.options['no-log']:
continue
entry = VersionLog.objects.create(package=version.package,
action=VersionLog.VERSION_REMOVED)
entry.slot = version.slot
entry.revision = version.revision
entry.version = version.version
entry.overlay = version.overlay
entry.save()
Version.objects.filter(packaged=True, alive=False).delete() Version.objects.filter(packaged=True, alive=False).delete()

View File

@ -10,6 +10,15 @@ from euscanwww.euscan.models import Package, Herd, Maintainer, Version
from euscanwww.euscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog from euscanwww.euscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from euscanwww.euscan import charts from euscanwww.euscan import charts
from distutils.version import StrictVersion, LooseVersion
def compare_versions(version1, version2):
try:
return cmp(StrictVersion(version1), StrictVersion(version2))
# in case of abnormal version number, fall back to LooseVersion
except ValueError:
return cmp(LooseVersion(version1), LooseVersion(version2))
class Command(BaseCommand): class Command(BaseCommand):
_overlays = {} _overlays = {}
help = 'Update counters' help = 'Update counters'
@ -88,7 +97,7 @@ class Command(BaseCommand):
return return
if version['version'].startswith('9999'): if version['version'].startswith('9999'):
return return
if storage[key]['version'] < version['version']: if compare_versions(storage[key]['version'], version['version']) < 0:
storage[key] = version storage[key] = version
if not options['fast']: if not options['fast']: