src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

euscanwww: try to keep trace of versions change

Browse Source 
Signed-off-by: Corentin Chary <corentincj@iksaif.net>
This commit is contained in:
Corentin Chary
2011-08-25 15:39:54 +02:00
parent 7570453bc2
commit d0fa19bc1d
15 changed files with 585 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
euscanwww/euscan/management/commands/scan-metadata.py
View File

@ -14,6 +14,8 @@ from euscanwww.euscan.models import Package, Herd, Maintainer
from gentoolkit.query import Query
from gentoolkit.errors import GentoolkitFatalError
from progressbar import ProgressBar, Bar, ETA, Percentage
class Command(BaseCommand):
_overlays = {}
@ -28,6 +30,11 @@ class Command(BaseCommand):
dest='quiet',
default=False,
help='Be quiet'),
make_option('--progress',
action='store_true',
dest='progress',
default=False,
help='Display progress'),
)
args = '<package package ...>'
help = 'Scans metadata and fills database'
@ -36,24 +43,41 @@ class Command(BaseCommand):
if len(args) == 0 and options['all'] == False:
raise CommandError('You must specify a package or use --all')
if not options['quiet']:
self.stdout.write('Scanning metadata...\n')
if not self.stdout.isatty():
options['progress'] = False
if options['progress']:
widgets = ['Scanning metadata: ', Percentage(), ' ', Bar(), ' ', ETA()]
if len(args):
count = len(args)
else:
count = Package.objects.count()
pbar = ProgressBar(widgets=widgets, maxval=count).start()
i = 0
else:
pbar = None
if len(args) == 0:
for pkg in Package.objects.all():
self.scan(options, '%s/%s' % (pkg.category, pkg.name))
if pbar:
pbar.update(i)
i += 1
else:
for package in args:
self.scan(options, package)
if pbar:
pbar.update(i)
i += 1
if not options['quiet']:
self.stdout.write('Done.\n')
if pbar:
pbar.finish()
@commit_on_success
def scan(self, options, query=None):
matches = Query(query).find(
include_masked=True,
in_installed=False
in_installed=False,
)
if not matches:
@ -73,6 +97,8 @@ class Command(BaseCommand):
except GentoolkitFatalError, err:
sys.stderr.write(self.style.ERROR("Gentoolkit fatal error: '%s'\n" % str(err)))
if created and not options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
if pkg.metadata:
obj.herds.clear()
obj.maintainers.clear()
@ -85,9 +111,6 @@ class Command(BaseCommand):
maintainer = self.store_maintainer(options, maintainer.name, maintainer.email)
obj.maintainers.add(maintainer)
if not options['quiet']:
sys.stdout.write('[p] %s/%s\n' % (pkg.category, pkg.name))
obj.save()
def store_herd(self, options, name, email):
@ -99,7 +122,7 @@ class Command(BaseCommand):
if created or herd.email != email:
if not options['quiet']:
sys.stdout.write('[h] %s <%s>\n' % (name, email))
sys.stdout.write('+ [h] %s <%s>\n' % (name, email))
herd.email = email
herd.save()
@ -116,7 +139,7 @@ class Command(BaseCommand):
if created:
if not options['quiet']:
sys.stdout.write('[m] %s <%s>\n' % (name.encode('utf-8'), email))
sys.stdout.write('+ [m] %s <%s>\n' % (name.encode('utf-8'), email))
maintainer.name = name
maintainer.save()

86
euscanwww/euscan/management/commands/scan-portage.py
View File

@ -9,7 +9,7 @@ from optparse import make_option
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from euscanwww.euscan.models import Package, Version
from euscanwww.euscan.models import Package, Version, VersionLog
class Command(BaseCommand):
_overlays = {}
@ -20,11 +20,16 @@ class Command(BaseCommand):
dest='all',
default=False,
help='Scan all packages'),
make_option('--purge',
make_option('--purge-packages',
action='store_true',
dest='purge',
dest='purge-packages',
default=False,
help='Purge old packages'),
make_option('--purge-versions',
action='store_true',
dest='purge-versions',
default=False,
help='Purge old versions'),
make_option('--quiet',
action='store_true',
dest='quiet',
@ -47,6 +52,9 @@ class Command(BaseCommand):
for package in args:
self.scan(options, package)
if options['purge-versions']:
self.purge_versions(options)
if not options['quiet']:
self.stdout.write('Done.\n')
@ -90,9 +98,9 @@ class Command(BaseCommand):
if len(output) == 0:
if not query:
return
if options['purge']:
if options['purge-packages']:
if not options['quiet']:
sys.stdout.write('[gc] %s\n' % (query))
sys.stdout.write('- [p] %s\n' % (query))
if '/' in query:
cat, pkg = portage.catsplit(query)
Package.objects.filter(category=cat, name=pkg).delete()
@ -128,12 +136,12 @@ class Command(BaseCommand):
self.store_version(options, package, cpv, slot, overlay)
if options['purge'] and not query:
if options['purge-packages'] and not query:
for package in Package.objects.all():
cp = "%s/%s" % (package.category, package.name)
if cp not in packages:
if not options['quiet']:
sys.stdout.write('[gc] %s\n' % (cp))
sys.stdout.write('- [p] %s\n' % (package))
package.delete()
def store_package(self, options, cat, pkg):
@ -141,10 +149,10 @@ class Command(BaseCommand):
if created:
if not options['quiet']:
sys.stdout.write('[p] %s/%s\n' % (cat, pkg))
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
# Delete previous versions to handle incremental scan correctly
Version.objects.filter(package=obj, packaged=True).delete()
' Set all versions dead, then set found versions alive and delete old versions '
Version.objects.filter(package=obj, packaged=True).update(alive=False)
obj.n_packaged = 0
obj.n_overlay = 0
@ -163,23 +171,53 @@ class Command(BaseCommand):
else:
overlay = 'gentoo'
if not options['quiet']:
sys.stdout.write('[v] %s:%s [%s]\n' % (cpv, slot, overlay))
obj, created = Version.objects.get_or_create(package=package, slot=slot,
revision=rev, version=ver,
overlay=overlay)
if created or not package.n_packaged:
if overlay == 'gentoo':
package.n_packaged += 1
else:
package.n_overlay += 1
if created:
package.n_versions += 1
package.save()
obj.alive = True
obj.packaged = True
obj.save()
''' nothing to do (note: it can't be an upstream version because overlay can't be empty here) '''
if not created:
return
if not options['quiet']:
sys.stdout.write('+ [v] %s \n' % (obj))
if overlay == 'gentoo':
package.n_packaged += 1
else:
package.n_overlay += 1
package.n_versions += 1
package.save()
entry = VersionLog.objects.create(package=obj.package, action=VersionLog.VERSION_ADDED)
entry.slot = obj.slot
entry.revision = obj.revision
entry.version = obj.version
entry.overlay = obj.overlay
entry.save()
@commit_on_success
def purge_versions(self, options):
' For each dead versions '
for version in Version.objects.filter(packaged=True, alive=False):
entry = VersionLog.objects.create(package=version.package, action=VersionLog.VERSION_REMOVED)
entry.slot = version.slot
entry.revision = version.revision
entry.version = version.version
entry.overlay = version.overlay
entry.save()
if version.overlay == 'gentoo':
version.package.n_packaged -= 1
else:
version.package.n_overlay -= 1
version.package.n_versions -= 1
version.package.save()
if not options['quiet']:
sys.stdout.write('- [v] %s\n' % (version))
Version.objects.filter(packaged=False, alive=False).delete()

72
euscanwww/euscan/management/commands/scan-upstream.py
View File

@ -11,7 +11,7 @@ from optparse import make_option
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from euscanwww.euscan.models import Package, Version, EuscanResult
from euscanwww.euscan.models import Package, Version, EuscanResult, VersionLog
class Command(BaseCommand):
_overlays = {}
@ -27,6 +27,11 @@ class Command(BaseCommand):
dest='feed',
default=False,
help='Read euscan output from stdin'),
make_option('--purge-versions',
action='store_true',
dest='purge-versions',
default=False,
help='Purge old versions'),
make_option('--quiet',
action='store_true',
dest='quiet',
@ -42,6 +47,8 @@ class Command(BaseCommand):
if options['feed']:
self.parse_output(options, sys.stdin)
if options['purge-versions']:
self.purge_versions(options)
return
if not options['quiet']:
@ -57,10 +64,12 @@ class Command(BaseCommand):
self.scan(options, packages)
if options['purge-versions']:
self.purge_versions(options)
if not options['quiet']:
self.stdout.write('Done.\n')
@commit_on_success
def scan(self, options, packages=None):
for package in packages:
cmd = ['euscan', package]
@ -70,6 +79,7 @@ class Command(BaseCommand):
self.parse_output(options, output)
@commit_on_success
def parse_output(self, options, output):
from portage.versions import _cp
@ -120,12 +130,11 @@ class Command(BaseCommand):
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
if created:
if not options['quiet']:
sys.stdout.write('[p] %s/%s\n' % (cat, pkg))
if created and not options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
# Delete previous versions to handle incremental scan correctly
Version.objects.filter(package=obj, packaged=False).delete()
' Set all versions dead, then set found versions alive and delete old versions '
Version.objects.filter(package=obj, packaged=False).update(alive=False)
obj.n_versions = Version.objects.filter(package=obj).count()
obj.save()
@ -137,15 +146,44 @@ class Command(BaseCommand):
revision='r0', version=ver,
overlay='')
if created or not obj.packaged:
obj.alive = True
obj.urls = url
obj.packaged = False
obj.save()
''' If it's not a new version, just update the object and continue '''
if not created:
return
if not options['quiet']:
sys.stdout.write('+ [u] %s %s\n' % (obj, url))
entry = VersionLog.objects.create(package=package, action=VersionLog.VERSION_ADDED)
entry.slot = ''
entry.revision = 'r0'
entry.version = ver
entry.overlay = ''
entry.save()
package.n_versions += 1
package.save()
@commit_on_success
def purge_versions(self, options):
' For each dead versions '
for version in Version.objects.filter(packaged=False, alive=False):
entry = VersionLog.objects.create(package=version.package, action=VersionLog.VERSION_REMOVED)
entry.slot = version.slot
entry.revision = version.revision
entry.version = version.version
entry.overlay = version.overlay
entry.save()
version.package.n_versions -= 1
version.package.save()
if not options['quiet']:
sys.stdout.write('[u] %s/%s-%s %s\n' % (package.category, package.name,
ver, url))
sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
Version.objects.filter(packaged=False, alive=False).delete()
obj.urls = url
obj.packaged = False
obj.save()
if created:
package.n_versions += 1
package.save()

12
euscanwww/euscan/management/commands/update-counters.py
View File

@ -29,8 +29,10 @@ class Command(BaseCommand):
herds = {}
maintainers = {}
# Could be done using raw SQL queries, but I don't have time for that
# right now ...
'''
Could be done using raw SQL queries, but I don't have time for that
right now ...
'''
wlog = WorldLog()
wlog.datetime = now
@ -100,19 +102,19 @@ class Command(BaseCommand):
for clog in categories.values():
if not options['quiet']:
self.stdout.write('[c] %s\n' % clog)
self.stdout.write('+ [cl] %s\n' % clog)
charts.rrd_update('category-%s' % clog.category, now, clog)
clog.save()
for hlog in herds.values():
if not options['quiet']:
self.stdout.write('[h] %s\n' % hlog)
self.stdout.write('+ [hl] %s\n' % hlog)
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
hlog.save()
for mlog in maintainers.values():
if not options['quiet']:
self.stdout.write('[m] %s\n' % mlog)
self.stdout.write('+ [ml] %s\n' % mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save()