euscanwww: add freshness and last versions
Signed-off-by: Corentin Chary <corentincj@iksaif.net>
This commit is contained in:
Corentin Chary
@ -5,6 +5,7 @@ from optparse import make_option
|
||||
from django.db.models import Count, Sum
|
||||
from django.db.transaction import commit_on_success
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from euscanwww.euscan.models import Package, Herd, Maintainer, Version
|
||||
from euscanwww.euscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
|
||||
from euscanwww.euscan import charts
|
||||
@ -19,6 +20,17 @@ class Command(BaseCommand):
|
||||
dest='quiet',
|
||||
default=False,
|
||||
help='Be quiet'),
|
||||
make_option('--fast',
|
||||
action='store_true',
|
||||
dest='fast',
|
||||
default=False,
|
||||
help='Skip sanity checks'),
|
||||
make_option('--nolog',
|
||||
action='store_true',
|
||||
dest='nolog',
|
||||
default=False,
|
||||
help='Skip logs'),
|
||||
|
||||
)
|
||||
|
||||
@commit_on_success
|
||||
@ -29,76 +41,118 @@ class Command(BaseCommand):
|
||||
herds = {}
|
||||
maintainers = {}
|
||||
|
||||
'''
|
||||
Could be done using raw SQL queries, but I don't have time for that
|
||||
right now ...
|
||||
'''
|
||||
wlog = None
|
||||
|
||||
wlog = WorldLog()
|
||||
wlog.datetime = now
|
||||
if not options['nolog']:
|
||||
wlog = WorldLog()
|
||||
wlog.datetime = now
|
||||
|
||||
for cat in Package.objects.values('category').distinct():
|
||||
clog = CategoryLog()
|
||||
clog.datetime = now
|
||||
clog.category = cat['category']
|
||||
categories[clog.category] = clog
|
||||
for cat in Package.objects.values('category').distinct():
|
||||
clog = CategoryLog()
|
||||
clog.datetime = now
|
||||
clog.category = cat['category']
|
||||
categories[clog.category] = clog
|
||||
|
||||
for herd in Herd.objects.all():
|
||||
hlog = HerdLog()
|
||||
hlog.datetime = now
|
||||
hlog.herd = herd
|
||||
herds[herd] = hlog
|
||||
for herd in Herd.objects.all():
|
||||
hlog = HerdLog()
|
||||
hlog.datetime = now
|
||||
hlog.herd = herd
|
||||
herds[herd.id] = hlog
|
||||
|
||||
for maintainer in Maintainer.objects.all():
|
||||
mlog = MaintainerLog()
|
||||
mlog.datetime = now
|
||||
mlog.maintainer = maintainer
|
||||
maintainers[maintainer] = mlog
|
||||
for maintainer in Maintainer.objects.all():
|
||||
mlog = MaintainerLog()
|
||||
mlog.datetime = now
|
||||
mlog.maintainer = maintainer
|
||||
maintainers[maintainer.id] = mlog
|
||||
|
||||
for package in Package.objects.all():
|
||||
# Should not be needed, but can't hurt
|
||||
package.n_versions = Version.objects.filter(package=package).count()
|
||||
package.n_packaged = Version.objects.filter(package=package, packaged=True, overlay='gentoo').count()
|
||||
package.n_overlay = Version.objects.filter(package=package, packaged=True).exclude(overlay='gentoo').count()
|
||||
package.save()
|
||||
package_queryset = Package.objects.all()
|
||||
|
||||
n_versions = {}
|
||||
n_packaged = {}
|
||||
n_overlay = {}
|
||||
|
||||
last_versions_gentoo = {}
|
||||
last_versions_overlay = {}
|
||||
last_versions_upstream = {}
|
||||
|
||||
def add_safe(storage, key):
|
||||
if key not in storage:
|
||||
storage[key] = 1
|
||||
else:
|
||||
storage[key] += 1
|
||||
|
||||
def add_last_ver(storage, version):
|
||||
key = version['package_id']
|
||||
if key not in storage:
|
||||
storage[key] = version
|
||||
return
|
||||
if version['version'].startswith('9999'):
|
||||
return
|
||||
if storage[key]['version'] < version['version']:
|
||||
storage[key] = version
|
||||
|
||||
if not options['fast']:
|
||||
attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
|
||||
for version in Version.objects.all().values(*attrs):
|
||||
overlay, packaged = version['overlay'], version['packaged']
|
||||
package_id = version['package_id']
|
||||
|
||||
add_safe(n_versions, package_id)
|
||||
|
||||
if not packaged:
|
||||
add_last_ver(last_versions_upstream, version)
|
||||
continue
|
||||
if overlay == 'gentoo':
|
||||
add_safe(n_packaged, package_id)
|
||||
add_last_ver(last_versions_gentoo, version)
|
||||
else:
|
||||
add_safe(n_overlay, package_id)
|
||||
add_last_ver(last_versions_overlay, version)
|
||||
|
||||
for package in package_queryset.select_related('herds', 'maintainers'):
|
||||
if not options['fast']:
|
||||
package.n_versions = n_versions.get(package.id, 0)
|
||||
package.n_packaged = n_packaged.get(package.id, 0)
|
||||
package.n_overlay = n_overlay.get(package.id, 0)
|
||||
|
||||
default = {'id' : -1}
|
||||
package.last_version_gentoo_id = last_versions_gentoo.get(package.id, default)['id']
|
||||
package.last_version_overlay_id = last_versions_overlay.get(package.id, default)['id']
|
||||
package.last_version_upstream_id = last_versions_upstream.get(package.id, default)['id']
|
||||
|
||||
package.save()
|
||||
|
||||
n_packages_gentoo = int(package.n_packaged == package.n_versions)
|
||||
n_packages_overlay = int(package.n_overlay and package.n_packaged + package.n_overlay == package.n_versions)
|
||||
n_packages_outdated = int(package.n_packaged + package.n_overlay < package.n_versions)
|
||||
|
||||
for herd in package.herds.all():
|
||||
herds[herd].n_packages_gentoo += n_packages_gentoo
|
||||
herds[herd].n_packages_overlay += n_packages_overlay
|
||||
herds[herd].n_packages_outdated += n_packages_outdated
|
||||
def update_row(storage, key):
|
||||
storage[key].n_packages_gentoo += n_packages_gentoo
|
||||
storage[key].n_packages_overlay += n_packages_overlay
|
||||
storage[key].n_packages_outdated += n_packages_outdated
|
||||
|
||||
herds[herd].n_versions_gentoo += package.n_packaged
|
||||
herds[herd].n_versions_overlay += package.n_overlay
|
||||
herds[herd].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
|
||||
storage[key].n_versions_gentoo += package.n_packaged
|
||||
storage[key].n_versions_overlay += package.n_overlay
|
||||
storage[key].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
|
||||
def update_log(storage, qs):
|
||||
for row in qs:
|
||||
update_row(storage, row['id'])
|
||||
|
||||
for maintainer in package.maintainers.all():
|
||||
maintainers[maintainer].n_packages_gentoo += n_packages_gentoo
|
||||
maintainers[maintainer].n_packages_overlay += n_packages_overlay
|
||||
maintainers[maintainer].n_packages_outdated += n_packages_outdated
|
||||
if not options['nolog']:
|
||||
update_log(herds, package.herds.all().values('id'))
|
||||
update_log(maintainers, package.maintainers.all().values('id'))
|
||||
update_row(categories, package.category)
|
||||
|
||||
maintainers[maintainer].n_versions_gentoo += package.n_packaged
|
||||
maintainers[maintainer].n_versions_overlay += package.n_overlay
|
||||
maintainers[maintainer].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
|
||||
wlog.n_packages_gentoo += n_packages_gentoo
|
||||
wlog.n_packages_overlay += n_packages_overlay
|
||||
wlog.n_packages_outdated += n_packages_outdated
|
||||
|
||||
categories[package.category].n_packages_gentoo += n_packages_gentoo
|
||||
categories[package.category].n_packages_overlay += n_packages_overlay
|
||||
categories[package.category].n_packages_outdated += n_packages_outdated
|
||||
wlog.n_versions_gentoo += package.n_packaged
|
||||
wlog.n_versions_overlay += package.n_overlay
|
||||
wlog.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
|
||||
|
||||
categories[package.category].n_versions_gentoo += package.n_packaged
|
||||
categories[package.category].n_versions_overlay += package.n_overlay
|
||||
categories[package.category].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
|
||||
|
||||
wlog.n_packages_gentoo += n_packages_gentoo
|
||||
wlog.n_packages_overlay += n_packages_overlay
|
||||
wlog.n_packages_outdated += n_packages_outdated
|
||||
|
||||
wlog.n_versions_gentoo += package.n_packaged
|
||||
wlog.n_versions_overlay += package.n_overlay
|
||||
wlog.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
|
||||
if options['nolog']:
|
||||
return
|
||||
|
||||
for clog in categories.values():
|
||||
if not options['quiet']:
|
||||
@ -118,6 +172,5 @@ class Command(BaseCommand):
|
||||
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
|
||||
mlog.save()
|
||||
|
||||
wlog.save()
|
||||
|
||||
charts.rrd_update('world', now, wlog)
|
||||
wlog.save()
|
||||
|
||||
Reference in New Issue
Block a user