Merge remote branch 'fox/master'

This commit is contained in:
Corentin Chary 2012-05-29 09:21:44 +02:00
commit f0a3c34ac8
12 changed files with 638 additions and 563 deletions

View File

@ -1,56 +0,0 @@
from optparse import make_option
from django.core.management.base import BaseCommand
from djeuscan.models import Package
class Command(BaseCommand):
_overlays = {}
help = 'List packages'
option_list = BaseCommand.option_list + (
make_option('--after',
action='store',
dest='after',
default=False,
help='After package'),
make_option('--before',
action='store',
dest='before',
default=False,
help='Before package'),
make_option('--limit',
action='store',
dest='limit',
default=False,
help='limit'),
)
def handle(self, *args, **options):
after = None
before = None
if options['after']:
category, name = options['after'].split('/')
after = Package.objects.get(category=category, name=name)
if options['before']:
category, name = options['before'].split('/')
before = Package.objects.get(category=category, name=name)
packages = Package.objects
if after or before:
if after:
packages = packages.filter(id__gte=after.id)
if before:
packages = packages.filter(id__lte=before.id)
else:
packages = packages.all()
if options['limit']:
packages = packages[:int(options['limit'])]
for pkg in packages:
self.stdout.write('%s/%s\n' % (pkg.category, pkg.name))
self.stdout.close()

View File

@ -0,0 +1,64 @@
import sys
from optparse import make_option
from django.core.management.base import BaseCommand
from djeuscan.models import Package
def list_packages(stdout=None, **options):
if stdout is None:
stdout = sys.stdout
after = None
before = None
if options['after']:
category, name = options['after'].split('/')
after = Package.objects.get(category=category, name=name)
if options['before']:
category, name = options['before'].split('/')
before = Package.objects.get(category=category, name=name)
packages = Package.objects
if after or before:
if after:
packages = packages.filter(id__gte=after.id)
if before:
packages = packages.filter(id__lte=before.id)
else:
packages = packages.all()
if options['limit']:
packages = packages[:int(options['limit'])]
for pkg in packages:
stdout.write('%s/%s\n' % (pkg.category, pkg.name))
stdout.close()
class Command(BaseCommand):
_overlays = {}
help = 'List packages'
option_list = BaseCommand.option_list + (
make_option('--after',
action='store',
dest='after',
default=False,
help='After package'),
make_option('--before',
action='store',
dest='before',
default=False,
help='Before package'),
make_option('--limit',
action='store',
dest='limit',
default=False,
help='limit'),
)
def handle(self, *args, **options):
list_packages(self.stdout, **options)

View File

@ -1,23 +0,0 @@
from django.core.management.base import BaseCommand
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from djeuscan import charts
class Command(BaseCommand):
_overlays = {}
help = 'Regenerate rrd database'
def handle(self, *args, **options):
for wlog in WorldLog.objects.all():
charts.rrd_update('world', wlog.datetime, wlog)
for clog in CategoryLog.objects.all():
charts.rrd_update('category-%s' % clog.category,
clog.datetime, clog)
for hlog in HerdLog.objects.all():
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
for mlog in MaintainerLog.objects.all():
charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
mlog.datetime, mlog)

View File

@ -0,0 +1,30 @@
from django.core.management.base import BaseCommand
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from djeuscan import charts
def regen_rrds():
"""
Regenerates the rrd database
"""
for wlog in WorldLog.objects.all():
charts.rrd_update('world', wlog.datetime, wlog)
for clog in CategoryLog.objects.all():
charts.rrd_update('category-%s' % clog.category,
clog.datetime, clog)
for hlog in HerdLog.objects.all():
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
for mlog in MaintainerLog.objects.all():
charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
mlog.datetime, mlog)
class Command(BaseCommand):
_overlays = {}
help = 'Regenerate rrd database'
def handle(self, *args, **options):
regen_rrds()

View File

@ -2,47 +2,23 @@ import sys
from optparse import make_option
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand
from djeuscan.models import Package, Herd, Maintainer
from gentoolkit.query import Query
from gentoolkit.errors import GentoolkitFatalError
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand
from django.core.management.color import color_style
class Command(BaseCommand):
_overlays = {}
from djeuscan.models import Package, Herd, Maintainer
option_list = BaseCommand.option_list + (
make_option('--all',
action='store_true',
dest='all',
default=False,
help='Scan all packages'),
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
)
args = '<package package ...>'
help = 'Scans metadata and fills database'
class ScanMetadata(object):
def __init__(self, quiet):
self.quiet = quiet
self.style = color_style()
@commit_on_success
def handle(self, *args, **options):
self.options = options
if options['all']:
for pkg in Package.objects.all():
self.scan('%s/%s' % (pkg.category, pkg.name), pkg)
elif len(args):
for package in args:
self.scan(package)
else:
for package in sys.stdin.readlines():
self.scan(package[:-1])
def scan(self, query=None, obj=None):
def run(self, query=None, obj=None):
matches = Query(query).find(
include_masked=True,
in_installed=False,
@ -76,7 +52,7 @@ class Command(BaseCommand):
)
)
if created and not self.options['quiet']:
if created and not self.quiet:
sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
if pkg.metadata:
@ -125,9 +101,12 @@ class Command(BaseCommand):
name = '{nil}'
name = name.strip("\r").strip("\n").strip("\t").strip()
herd, created = Herd.objects.get_or_create(herd=name)
herd, created = Herd.objects.get_or_create(
herd=name,
defaults={"email": email}
)
if created and not self.options['quiet']:
if created and not self.quiet:
sys.stdout.write('+ [h] %s <%s>\n' % (name, email))
herd.email = email
@ -141,17 +120,48 @@ class Command(BaseCommand):
if not name:
name = '{nil}'
maintainer, created = Maintainer.objects.get_or_create(email=email)
maintainer, created = Maintainer.objects.get_or_create(
email=email,
defaults={"name": name}
)
if created:
if not self.options['quiet']:
if not self.quiet:
sys.stdout.write(
'+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
)
if not maintainer.name or \
name not in [maintainer.name, email, '{nil}']:
maintainer.name = name
maintainer.save()
return maintainer
class Command(BaseCommand):
_overlays = {}
option_list = BaseCommand.option_list + (
make_option('--all',
action='store_true',
dest='all',
default=False,
help='Scan all packages'),
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
)
args = '<package package ...>'
help = 'Scans metadata and fills database'
def handle(self, *args, **options):
self.options = options
scan_metadata = ScanMetadata(quiet=options["quiet"])
if options['all']:
for pkg in Package.objects.all():
scan_metadata.run('%s/%s' % (pkg.category, pkg.name), pkg)
elif len(args) > 0:
for package in args:
scan_metadata.run(package)
else:
for package in sys.stdin.readlines():
scan_metadata.run(package[:-1])

View File

@ -8,49 +8,22 @@ from optparse import make_option
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand
from django.core.management.color import color_style
from djeuscan.models import Package, Version, VersionLog
class Command(BaseCommand):
_overlays = {}
class ScanPortage(object):
def __init__(self, stdout=None, **options):
if stdout is None:
self.stdout = sys.stdout
else:
self.stdout = stdout
option_list = BaseCommand.option_list + (
make_option('--all',
action='store_true',
dest='all',
default=False,
help='Scan all packages'),
make_option('--purge-packages',
action='store_true',
dest='purge-packages',
default=False,
help='Purge old packages'),
make_option('--purge-versions',
action='store_true',
dest='purge-versions',
default=False,
help='Purge old versions'),
make_option('--no-log',
action='store_true',
dest='no-log',
default=False,
help='Don\'t store logs'),
make_option('--prefetch',
action='store_true',
dest='prefetch',
default=False,
help=('Prefetch all versions and packages from DB to '
'speedup full scan process.')),
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
)
args = '[package package ...]'
help = 'Scans portage tree and fills database'
_cache = {'packages': {}, 'versions': {}}
self.options = options
self.style = color_style()
self._cache = {'packages': {}, 'versions': {}}
self._overlays = None
def cache_hash_package(self, category, name):
return '%s/%s' % (category, name)
@ -84,38 +57,6 @@ class Command(BaseCommand):
)
self._cache['versions'][key] = version
def handle(self, *args, **options):
self.options = options
if not options['quiet']:
self.stdout.write('Scanning portage tree...\n')
if options['prefetch']:
if not options['quiet']:
self.stdout.write('Prefetching objects...')
self.stdout.flush()
for package in Package.objects.all():
self.cache_store_package(package)
for version in Version.objects.select_related('package').all():
self.cache_store_version(version)
if not options['quiet']:
self.stdout.write('done\n')
if options['all']:
self.scan()
elif len(args):
for package in args:
self.scan(package)
else:
for package in sys.stdin.readlines():
self.scan(package[:-1])
if options['purge-versions']:
self.purge_versions(options)
if not options['quiet']:
self.stdout.write('Done.\n')
def overlays(self):
if self._overlays:
return self._overlays
@ -143,7 +84,7 @@ class Command(BaseCommand):
return self._overlays
@commit_on_success
def scan(self, query=None):
def run(self, query=None):
env = os.environ
env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
@ -263,9 +204,10 @@ class Command(BaseCommand):
obj, created = Version.objects.get_or_create(
package=package, slot=slot,
revision=rev, version=ver,
overlay=overlay
overlay=overlay,
defaults={"alive": True, "packaged": True}
)
if not created:
obj.alive = True
obj.packaged = True
obj.save()
@ -291,17 +233,19 @@ class Command(BaseCommand):
if self.options['no-log']:
return
entry = VersionLog.objects.create(package=obj.package,
action=VersionLog.VERSION_ADDED)
entry.slot = obj.slot
entry.revision = obj.revision
entry.version = obj.version
entry.overlay = obj.overlay
entry.save()
VersionLog.objects.create(
package=obj.package,
action=VersionLog.VERSION_ADDED,
slot=obj.slot,
revision=obj.revision,
version=obj.version,
overlay=obj.overlay
)
@commit_on_success
def purge_versions(self, options):
' For each dead versions '
def purge_versions(options):
# For each dead versions
for version in Version.objects.filter(packaged=True, alive=False):
if version.overlay == 'gentoo':
version.package.n_packaged -= 1
@ -310,20 +254,91 @@ class Command(BaseCommand):
version.package.n_versions -= 1
version.package.save()
if not self.options['quiet']:
if not options['quiet']:
sys.stdout.write('- [v] %s\n' % (version))
if self.options['no-log']:
if options['no-log']:
continue
entry = VersionLog.objects.create(
VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED
action=VersionLog.VERSION_REMOVED,
slot=version.slot,
revision=version.revision,
version=version.version,
overlay=version.overlay
)
entry.slot = version.slot
entry.revision = version.revision
entry.version = version.version
entry.overlay = version.overlay
entry.save()
Version.objects.filter(packaged=True, alive=False).delete()
class Command(BaseCommand):
_overlays = {}
option_list = BaseCommand.option_list + (
make_option('--all',
action='store_true',
dest='all',
default=False,
help='Scan all packages'),
make_option('--purge-packages',
action='store_true',
dest='purge-packages',
default=False,
help='Purge old packages'),
make_option('--purge-versions',
action='store_true',
dest='purge-versions',
default=False,
help='Purge old versions'),
make_option('--no-log',
action='store_true',
dest='no-log',
default=False,
help='Don\'t store logs'),
make_option('--prefetch',
action='store_true',
dest='prefetch',
default=False,
help=('Prefetch all versions and packages from DB to '
'speedup full scan process.')),
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
)
args = '[package package ...]'
help = 'Scans portage tree and fills database'
def handle(self, *args, **options):
scan_portage = ScanPortage(stdout=self.stdout, **options)
if not options['quiet']:
self.stdout.write('Scanning portage tree...\n')
if options['prefetch']:
if not options['quiet']:
self.stdout.write('Prefetching objects...')
self.stdout.flush()
for package in Package.objects.all():
scan_portage.cache_store_package(package)
for version in Version.objects.select_related('package').all():
scan_portage.cache_store_version(version)
if not options['quiet']:
self.stdout.write('done\n')
if options['all']:
scan_portage.run()
elif len(args):
for package in args:
scan_portage.run(package)
else:
for package in sys.stdin.readlines():
scan_portage.run(package[:-1])
if options['purge-versions']:
purge_versions(options)
if not options['quiet']:
self.stdout.write('Done.\n')

View File

@ -2,16 +2,149 @@ import subprocess
import portage
import sys
import re
from StringIO import StringIO
from optparse import make_option
from collections import defaultdict
from django.utils import timezone
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand
from djeuscan.models import Package, Version, EuscanResult, VersionLog
class ScanUpstream(object):
def __init__(self, options=None):
if options is None:
self.options = defaultdict(None)
else:
self.options = options
def run(self, packages=None):
for package in packages:
cmd = ['euscan', package]
fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = StringIO(fp.communicate()[0])
self.parse_output(output)
def store_result(self, package, log):
# Remove previous logs
EuscanResult.objects.filter(package=package).delete()
obj = EuscanResult()
obj.package = package
obj.result = log
obj.datetime = timezone.now()
obj.save()
def store_package(self, cpv):
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
if created and not self.options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
# Set all versions dead, then set found versions alive and
# delete old versions
Version.objects.filter(package=obj, packaged=False).update(alive=False)
return obj
def store_version(self, package, ver, url):
obj, created = Version.objects.get_or_create(
package=package, slot='', revision='r0', version=ver, overlay='',
defaults={"alive": True, "urls": url, "packaged": True}
)
if not created:
obj.alive = True
obj.urls = url
obj.packaged = False
obj.save()
# If it's not a new version, just update the object and continue
if not created:
return
if not self.options['quiet']:
sys.stdout.write('+ [u] %s %s\n' % (obj, url))
VersionLog.objects.create(
package=package,
action=VersionLog.VERSION_ADDED,
slot='',
revision='r0',
version=ver,
overlay=''
)
package.n_versions += 1
package.save()
@commit_on_success
def parse_output(self, output):
from portage.versions import _cp
package_re = re.compile(
r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$'
)
version_re = re.compile(
r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$'
)
package = None
log = ""
while True:
line = output.readline()
if line == '':
break
match = package_re.match(line)
if match:
if package:
self.store_result(package, log)
cpv = match.group('cpv')
package = self.store_package(cpv)
log = line
continue
log += line
match = version_re.match(line)
if match:
ver = match.group('ver')
url = match.group('url')
self.store_version(package, ver, url)
if package:
self.store_result(package, log)
@commit_on_success
def purge_versions(options):
# For each dead versions
for version in Version.objects.filter(packaged=False, alive=False):
VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED,
slot=version.slot,
revision=version.revision,
version=version.version,
overlay=version.overlay
)
version.package.n_versions -= 1
version.package.save()
if not options['quiet']:
sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
Version.objects.filter(packaged=False, alive=False).delete()
class Command(BaseCommand):
_overlays = {}
@ -41,10 +174,12 @@ class Command(BaseCommand):
help = 'Scans metadata and fills database'
def handle(self, *args, **options):
scan_upstream = ScanUpstream(options)
if options['feed']:
self.parse_output(options, sys.stdin)
scan_upstream.parse_output(options, sys.stdin)
if options['purge-versions']:
self.purge_versions(options)
purge_versions(options)
return
if not options['quiet']:
@ -60,133 +195,10 @@ class Command(BaseCommand):
else:
packages = [package[:-1] for package in sys.stdin.readlines()]
self.scan(options, packages)
scan_upstream.run(packages)
if options['purge-versions']:
self.purge_versions(options)
purge_versions(options)
if not options['quiet']:
self.stdout.write('Done.\n')
def scan(self, options, packages=None):
for package in packages:
cmd = ['euscan', package]
fp = subprocess.Popen(cmd, stdout=subprocess.PIPE)
output = StringIO(fp.communicate()[0])
self.parse_output(options, output)
@commit_on_success
def parse_output(self, options, output):
from portage.versions import _cp
package_re = re.compile(
r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$'
)
version_re = re.compile(
r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$'
)
package = None
log = ""
while True:
line = output.readline()
if line == '':
break
match = package_re.match(line)
if match:
if package:
self.store_result(options, package, log)
cpv = match.group('cpv')
package = self.store_package(options, cpv)
log = line
continue
log += line
match = version_re.match(line)
if match:
ver = match.group('ver')
url = match.group('url')
self.store_version(options, package, ver, url)
if package:
self.store_result(options, package, log)
def store_result(self, options, package, log):
# Remove previous logs
EuscanResult.objects.filter(package=package).delete()
obj = EuscanResult()
obj.package = package
obj.result = log
obj.datetime = timezone.now()
obj.save()
def store_package(self, options, cpv):
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
if created and not options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
# Set all versions dead, then set found versions alive and
# delete old versions
Version.objects.filter(package=obj, packaged=False).update(alive=False)
return obj
def store_version(self, options, package, ver, url):
obj, created = Version.objects.get_or_create(
package=package, slot='', revision='r0', version=ver, overlay=''
)
obj.alive = True
obj.urls = url
obj.packaged = False
obj.save()
''' If it's not a new version, just update the object and continue '''
if not created:
return
if not options['quiet']:
sys.stdout.write('+ [u] %s %s\n' % (obj, url))
entry = VersionLog.objects.create(
package=package,
action=VersionLog.VERSION_ADDED
)
entry.slot = ''
entry.revision = 'r0'
entry.version = ver
entry.overlay = ''
entry.save()
package.n_versions += 1
package.save()
@commit_on_success
def purge_versions(self, options):
' For each dead versions '
for version in Version.objects.filter(packaged=False, alive=False):
entry = VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED
)
entry.slot = version.slot
entry.revision = version.revision
entry.version = version.version
entry.overlay = version.overlay
entry.save()
version.package.n_versions -= 1
version.package.save()
if not options['quiet']:
sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
Version.objects.filter(packaged=False, alive=False).delete()

View File

@ -1,199 +0,0 @@
from optparse import make_option
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand
from django.utils import timezone
from djeuscan.models import Package, Herd, Maintainer, Version
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from djeuscan import charts
from distutils.version import StrictVersion, LooseVersion
def compare_versions(version1, version2):
try:
return cmp(StrictVersion(version1), StrictVersion(version2))
# in case of abnormal version number, fall back to LooseVersion
except ValueError:
return cmp(LooseVersion(version1), LooseVersion(version2))
class Command(BaseCommand):
_overlays = {}
help = 'Update counters'
option_list = BaseCommand.option_list + (
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
make_option('--fast',
action='store_true',
dest='fast',
default=False,
help='Skip sanity checks'),
make_option('--nolog',
action='store_true',
dest='nolog',
default=False,
help='Skip logs'),
)
@commit_on_success
def handle(self, *args, **options):
now = timezone.now()
categories = {}
herds = {}
maintainers = {}
wlog = None
if not options['nolog']:
wlog = WorldLog()
wlog.datetime = now
for cat in Package.objects.values('category').distinct():
clog = CategoryLog()
clog.datetime = now
clog.category = cat['category']
categories[clog.category] = clog
for herd in Herd.objects.all():
hlog = HerdLog()
hlog.datetime = now
hlog.herd = herd
herds[herd.id] = hlog
for maintainer in Maintainer.objects.all():
mlog = MaintainerLog()
mlog.datetime = now
mlog.maintainer = maintainer
maintainers[maintainer.id] = mlog
package_queryset = Package.objects.all()
n_versions = {}
n_packaged = {}
n_overlay = {}
last_versions_gentoo = {}
last_versions_overlay = {}
last_versions_upstream = {}
def add_safe(storage, key):
if key not in storage:
storage[key] = 1
else:
storage[key] += 1
def add_last_ver(storage, version):
key = version['package_id']
if key not in storage:
storage[key] = version
return
if version['version'].startswith('9999'):
return
if compare_versions(storage[key]['version'],
version['version']) < 0:
storage[key] = version
if not options['fast']:
attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
for version in Version.objects.all().values(*attrs):
overlay, packaged = version['overlay'], version['packaged']
package_id = version['package_id']
add_safe(n_versions, package_id)
if not packaged:
add_last_ver(last_versions_upstream, version)
continue
if overlay == 'gentoo':
add_safe(n_packaged, package_id)
add_last_ver(last_versions_gentoo, version)
else:
add_safe(n_overlay, package_id)
add_last_ver(last_versions_overlay, version)
for package in package_queryset.select_related('herds', 'maintainers'):
if not options['fast']:
package.n_versions = n_versions.get(package.id, 0)
package.n_packaged = n_packaged.get(package.id, 0)
package.n_overlay = n_overlay.get(package.id, 0)
default = {'id': None}
package.last_version_gentoo_id = last_versions_gentoo.get(
package.id, default
)['id']
package.last_version_overlay_id = last_versions_overlay.get(
package.id, default
)['id']
package.last_version_upstream_id = last_versions_upstream.get(
package.id, default
)['id']
package.save()
n_packages_gentoo = int(package.n_packaged == package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged \
+ package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay \
< package.n_versions)
def update_row(storage, key):
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_outdated += n_packages_outdated
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
def update_log(storage, qs):
for row in qs:
update_row(storage, row['id'])
if not options['nolog']:
update_log(herds, package.herds.all().values('id'))
update_log(maintainers, package.maintainers.all().values('id'))
update_row(categories, package.category)
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
if options['nolog']:
return
for clog in categories.values():
if not options['quiet']:
self.stdout.write('+ [cl] %s\n' % clog)
charts.rrd_update('category-%s' % clog.category, now, clog)
clog.save()
for hlog in herds.values():
if not options['quiet']:
self.stdout.write('+ [hl] %s\n' % hlog)
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
hlog.save()
for mlog in maintainers.values():
if not options['quiet']:
self.stdout.write('+ [ml] %s\n' % mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save()
charts.rrd_update('world', now, wlog)
wlog.save()

View File

@ -0,0 +1,208 @@
import sys
from optparse import make_option
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand
from django.utils import timezone
from djeuscan.models import Package, Herd, Maintainer, Version
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from djeuscan import charts
from distutils.version import StrictVersion, LooseVersion
def compare_versions(version1, version2):
try:
return cmp(StrictVersion(version1), StrictVersion(version2))
# in case of abnormal version number, fall back to LooseVersion
except ValueError:
return cmp(LooseVersion(version1), LooseVersion(version2))
def add_safe(storage, key):
if key not in storage:
storage[key] = 1
else:
storage[key] += 1
def add_last_ver(storage, version):
key = version['package_id']
if key not in storage:
storage[key] = version
return
if version['version'].startswith('9999'):
return
if compare_versions(storage[key]['version'],
version['version']) < 0:
storage[key] = version
@commit_on_success
def update_counters(stdout=None, **options):
if stdout is None:
stdout = sys.stdout
now = timezone.now()
categories = {}
herds = {}
maintainers = {}
wlog = None
if not options['nolog']:
wlog = WorldLog()
wlog.datetime = now
for cat in Package.objects.values('category').distinct():
clog = CategoryLog()
clog.datetime = now
clog.category = cat['category']
categories[clog.category] = clog
for herd in Herd.objects.all():
hlog = HerdLog()
hlog.datetime = now
hlog.herd = herd
herds[herd.id] = hlog
for maintainer in Maintainer.objects.all():
mlog = MaintainerLog()
mlog.datetime = now
mlog.maintainer = maintainer
maintainers[maintainer.id] = mlog
package_queryset = Package.objects.all()
n_versions = {}
n_packaged = {}
n_overlay = {}
last_versions_gentoo = {}
last_versions_overlay = {}
last_versions_upstream = {}
if not options['fast']:
attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
for version in Version.objects.all().values(*attrs):
overlay, packaged = version['overlay'], version['packaged']
package_id = version['package_id']
add_safe(n_versions, package_id)
if not packaged:
add_last_ver(last_versions_upstream, version)
continue
if overlay == 'gentoo':
add_safe(n_packaged, package_id)
add_last_ver(last_versions_gentoo, version)
else:
add_safe(n_overlay, package_id)
add_last_ver(last_versions_overlay, version)
for package in package_queryset.select_related('herds', 'maintainers'):
if not options['fast']:
package.n_versions = n_versions.get(package.id, 0)
package.n_packaged = n_packaged.get(package.id, 0)
package.n_overlay = n_overlay.get(package.id, 0)
default = {'id': None}
package.last_version_gentoo_id = last_versions_gentoo.get(
package.id, default
)['id']
package.last_version_overlay_id = last_versions_overlay.get(
package.id, default
)['id']
package.last_version_upstream_id = last_versions_upstream.get(
package.id, default
)['id']
package.save()
n_packages_gentoo = int(package.n_packaged == package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged \
+ package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay \
< package.n_versions)
def update_row(storage, key):
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_outdated += n_packages_outdated
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
def update_log(storage, qs):
for row in qs:
update_row(storage, row['id'])
if not options['nolog']:
update_log(herds, package.herds.all().values('id'))
update_log(maintainers, package.maintainers.all().values('id'))
update_row(categories, package.category)
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
if options['nolog']:
return
for clog in categories.values():
if not options['quiet']:
stdout.write('+ [cl] %s\n' % clog)
charts.rrd_update('category-%s' % clog.category, now, clog)
clog.save()
for hlog in herds.values():
if not options['quiet']:
stdout.write('+ [hl] %s\n' % hlog)
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
hlog.save()
for mlog in maintainers.values():
if not options['quiet']:
stdout.write('+ [ml] %s\n' % mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save()
charts.rrd_update('world', now, wlog)
wlog.save()
class Command(BaseCommand):
_overlays = {}
help = 'Update counters'
option_list = BaseCommand.option_list + (
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
make_option('--fast',
action='store_true',
dest='fast',
default=False,
help='Skip sanity checks'),
make_option('--nolog',
action='store_true',
dest='nolog',
default=False,
help='Skip logs'),
)
def handle(self, *args, **options):
update_counters(stdout=self.stdout, **options)

View File

@ -1,11 +1,12 @@
from django.db import models
from django.core.validators import RegexValidator, validate_email, URLValidator
from django.core.exceptions import ValidationError
from djeuscan.managers import PackageManager, VersionLogManager, \
EuscanResultManager
validate_category = RegexValidator("^\w+?-\w+?$")
validate_category = RegexValidator("^(?:\w+?-\w+?)|virtual$")
validate_name = RegexValidator("^\S+?$")
validate_revision = RegexValidator("^r\d+?$")
validate_url = URLValidator()
@ -56,7 +57,7 @@ class Package(models.Model):
category = models.CharField(max_length=128, validators=[validate_category])
name = models.CharField(max_length=128, validators=[validate_name])
description = models.TextField(blank=True)
homepage = models.TextField(blank=True, validators=[validate_url])
homepage = models.TextField(blank=True)
herds = models.ManyToManyField(Herd, blank=True)
maintainers = models.ManyToManyField(Maintainer, blank=True)
@ -89,6 +90,18 @@ class Package(models.Model):
def save(self, *args, **kwargs):
self.full_clean()
# Clean urls, accept only real urls
urls = []
for url in self.homepages:
try:
validate_url(url)
except ValidationError:
pass
else:
urls.append(url)
self.homepage = " ".join(urls)
super(Package, self).save(*args, **kwargs)
@property

View File

@ -87,8 +87,9 @@ setup(
'https://github.com/iksaif/euscan/tarball/' +
('master' if __version__ == '9999' else ('euscan-%s' % __version__))
),
install_requires=['Django==1.4', 'django-annoying', 'South',
'django-piston', 'BeautifulSoup', "matplotlib"],
install_requires=['Django==1.4', 'django-annoying==0..7.6', 'South==0.7.4',
'django-piston==0.2.3', 'BeautifulSoup==3.2.1',
'matplotlib==1.1.0', 'django-celery==2.5.5'],
package_dir={'': 'pym'},
packages=packages,
package_data={},