euscan: Refactored all management commands
Management commands are refactored to have the logic moved from the commands themselves to a separated function in order to make them easily convertible into Celery tasks Commands are renamed to be modules importable from other python scripts Signed-off-by: volpino <fox91@anche.no>
This commit is contained in:
volpino
parent
e535c204b0
commit
b46e9acc08
@ -1,56 +0,0 @@
|
||||
from optparse import make_option
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from djeuscan.models import Package
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
help = 'List packages'
|
||||
|
||||
option_list = BaseCommand.option_list + (
|
||||
make_option('--after',
|
||||
action='store',
|
||||
dest='after',
|
||||
default=False,
|
||||
help='After package'),
|
||||
make_option('--before',
|
||||
action='store',
|
||||
dest='before',
|
||||
default=False,
|
||||
help='Before package'),
|
||||
make_option('--limit',
|
||||
action='store',
|
||||
dest='limit',
|
||||
default=False,
|
||||
help='limit'),
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
after = None
|
||||
before = None
|
||||
|
||||
if options['after']:
|
||||
category, name = options['after'].split('/')
|
||||
after = Package.objects.get(category=category, name=name)
|
||||
|
||||
if options['before']:
|
||||
category, name = options['before'].split('/')
|
||||
before = Package.objects.get(category=category, name=name)
|
||||
|
||||
packages = Package.objects
|
||||
|
||||
if after or before:
|
||||
if after:
|
||||
packages = packages.filter(id__gte=after.id)
|
||||
if before:
|
||||
packages = packages.filter(id__lte=before.id)
|
||||
else:
|
||||
packages = packages.all()
|
||||
|
||||
if options['limit']:
|
||||
packages = packages[:int(options['limit'])]
|
||||
|
||||
for pkg in packages:
|
||||
self.stdout.write('%s/%s\n' % (pkg.category, pkg.name))
|
||||
self.stdout.close()
|
||||
64
euscanwww/djeuscan/management/commands/list_packages.py
Normal file
64
euscanwww/djeuscan/management/commands/list_packages.py
Normal file
@ -0,0 +1,64 @@
|
||||
import sys
|
||||
from optparse import make_option
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from djeuscan.models import Package
|
||||
|
||||
|
||||
def list_packages(stdout=None, **options):
|
||||
if stdout is None:
|
||||
stdout = sys.stdout
|
||||
|
||||
after = None
|
||||
before = None
|
||||
|
||||
if options['after']:
|
||||
category, name = options['after'].split('/')
|
||||
after = Package.objects.get(category=category, name=name)
|
||||
|
||||
if options['before']:
|
||||
category, name = options['before'].split('/')
|
||||
before = Package.objects.get(category=category, name=name)
|
||||
|
||||
packages = Package.objects
|
||||
|
||||
if after or before:
|
||||
if after:
|
||||
packages = packages.filter(id__gte=after.id)
|
||||
if before:
|
||||
packages = packages.filter(id__lte=before.id)
|
||||
else:
|
||||
packages = packages.all()
|
||||
|
||||
if options['limit']:
|
||||
packages = packages[:int(options['limit'])]
|
||||
|
||||
for pkg in packages:
|
||||
stdout.write('%s/%s\n' % (pkg.category, pkg.name))
|
||||
stdout.close()
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
help = 'List packages'
|
||||
|
||||
option_list = BaseCommand.option_list + (
|
||||
make_option('--after',
|
||||
action='store',
|
||||
dest='after',
|
||||
default=False,
|
||||
help='After package'),
|
||||
make_option('--before',
|
||||
action='store',
|
||||
dest='before',
|
||||
default=False,
|
||||
help='Before package'),
|
||||
make_option('--limit',
|
||||
action='store',
|
||||
dest='limit',
|
||||
default=False,
|
||||
help='limit'),
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
list_packages(self.stdout, **options)
|
||||
@ -1,23 +0,0 @@
|
||||
from django.core.management.base import BaseCommand
|
||||
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
|
||||
from djeuscan import charts
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
help = 'Regenerate rrd database'
|
||||
|
||||
def handle(self, *args, **options):
|
||||
for wlog in WorldLog.objects.all():
|
||||
charts.rrd_update('world', wlog.datetime, wlog)
|
||||
|
||||
for clog in CategoryLog.objects.all():
|
||||
charts.rrd_update('category-%s' % clog.category,
|
||||
clog.datetime, clog)
|
||||
|
||||
for hlog in HerdLog.objects.all():
|
||||
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
|
||||
|
||||
for mlog in MaintainerLog.objects.all():
|
||||
charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
|
||||
mlog.datetime, mlog)
|
||||
30
euscanwww/djeuscan/management/commands/regen_rrds.py
Normal file
30
euscanwww/djeuscan/management/commands/regen_rrds.py
Normal file
@ -0,0 +1,30 @@
|
||||
from django.core.management.base import BaseCommand
|
||||
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
|
||||
from djeuscan import charts
|
||||
|
||||
|
||||
def regen_rrds():
|
||||
"""
|
||||
Regenerates the rrd database
|
||||
"""
|
||||
for wlog in WorldLog.objects.all():
|
||||
charts.rrd_update('world', wlog.datetime, wlog)
|
||||
|
||||
for clog in CategoryLog.objects.all():
|
||||
charts.rrd_update('category-%s' % clog.category,
|
||||
clog.datetime, clog)
|
||||
|
||||
for hlog in HerdLog.objects.all():
|
||||
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
|
||||
|
||||
for mlog in MaintainerLog.objects.all():
|
||||
charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
|
||||
mlog.datetime, mlog)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
help = 'Regenerate rrd database'
|
||||
|
||||
def handle(self, *args, **options):
|
||||
regen_rrds()
|
||||
@ -2,47 +2,22 @@ import sys
|
||||
|
||||
from optparse import make_option
|
||||
|
||||
from django.db.transaction import commit_on_success
|
||||
from django.core.management.base import BaseCommand
|
||||
from djeuscan.models import Package, Herd, Maintainer
|
||||
|
||||
from gentoolkit.query import Query
|
||||
from gentoolkit.errors import GentoolkitFatalError
|
||||
|
||||
from django.db.transaction import commit_on_success
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.core.management.color import color_style
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
from djeuscan.models import Package, Herd, Maintainer
|
||||
|
||||
option_list = BaseCommand.option_list + (
|
||||
make_option('--all',
|
||||
action='store_true',
|
||||
dest='all',
|
||||
default=False,
|
||||
help='Scan all packages'),
|
||||
make_option('--quiet',
|
||||
action='store_true',
|
||||
dest='quiet',
|
||||
default=False,
|
||||
help='Be quiet'),
|
||||
)
|
||||
args = '<package package ...>'
|
||||
help = 'Scans metadata and fills database'
|
||||
|
||||
class ScanMetadata(object):
|
||||
def __init__(self, quiet):
|
||||
self.quiet = quiet
|
||||
|
||||
@commit_on_success
|
||||
def handle(self, *args, **options):
|
||||
self.options = options
|
||||
|
||||
if options['all']:
|
||||
for pkg in Package.objects.all():
|
||||
self.scan('%s/%s' % (pkg.category, pkg.name), pkg)
|
||||
elif len(args):
|
||||
for package in args:
|
||||
self.scan(package)
|
||||
else:
|
||||
for package in sys.stdin.readlines():
|
||||
self.scan(package[:-1])
|
||||
|
||||
def scan(self, query=None, obj=None):
|
||||
def run(self, query=None, obj=None):
|
||||
matches = Query(query).find(
|
||||
include_masked=True,
|
||||
in_installed=False,
|
||||
@ -50,7 +25,7 @@ class Command(BaseCommand):
|
||||
|
||||
if not matches:
|
||||
sys.stderr.write(
|
||||
self.style.ERROR("Unknown package '%s'\n" % query)
|
||||
color_style.ERROR("Unknown package '%s'\n" % query)
|
||||
)
|
||||
return
|
||||
|
||||
@ -71,12 +46,12 @@ class Command(BaseCommand):
|
||||
obj.description = pkg.environment("DESCRIPTION")
|
||||
except GentoolkitFatalError, err:
|
||||
sys.stderr.write(
|
||||
self.style.ERROR(
|
||||
color_style.ERROR(
|
||||
"Gentoolkit fatal error: '%s'\n" % str(err)
|
||||
)
|
||||
)
|
||||
|
||||
if created and not self.options['quiet']:
|
||||
if created and not self.quiet:
|
||||
sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
|
||||
|
||||
if pkg.metadata:
|
||||
@ -127,7 +102,7 @@ class Command(BaseCommand):
|
||||
|
||||
herd, created = Herd.objects.get_or_create(herd=name)
|
||||
|
||||
if created and not self.options['quiet']:
|
||||
if created and not self.quiet:
|
||||
sys.stdout.write('+ [h] %s <%s>\n' % (name, email))
|
||||
|
||||
herd.email = email
|
||||
@ -144,7 +119,7 @@ class Command(BaseCommand):
|
||||
maintainer, created = Maintainer.objects.get_or_create(email=email)
|
||||
|
||||
if created:
|
||||
if not self.options['quiet']:
|
||||
if not self.quiet:
|
||||
sys.stdout.write(
|
||||
'+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
|
||||
)
|
||||
@ -155,3 +130,37 @@ class Command(BaseCommand):
|
||||
maintainer.save()
|
||||
|
||||
return maintainer
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
|
||||
option_list = BaseCommand.option_list + (
|
||||
make_option('--all',
|
||||
action='store_true',
|
||||
dest='all',
|
||||
default=False,
|
||||
help='Scan all packages'),
|
||||
make_option('--quiet',
|
||||
action='store_true',
|
||||
dest='quiet',
|
||||
default=False,
|
||||
help='Be quiet'),
|
||||
)
|
||||
args = '<package package ...>'
|
||||
help = 'Scans metadata and fills database'
|
||||
|
||||
def handle(self, *args, **options):
|
||||
self.options = options
|
||||
|
||||
scan_metadata = ScanMetadata(quiet=options["quiet"])
|
||||
|
||||
if options['all']:
|
||||
for pkg in Package.objects.all():
|
||||
scan_metadata.run('%s/%s' % (pkg.category, pkg.name), pkg)
|
||||
elif len(args) > 0:
|
||||
for package in args:
|
||||
scan_metadata.run(package)
|
||||
else:
|
||||
for package in sys.stdin.readlines():
|
||||
scan_metadata.run(package[:-1])
|
||||
@ -8,49 +8,21 @@ from optparse import make_option
|
||||
|
||||
from django.db.transaction import commit_on_success
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.core.management.color import color_style
|
||||
|
||||
from djeuscan.models import Package, Version, VersionLog
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
class ScanPortage(object):
|
||||
def __init__(self, stdout=None, **options):
|
||||
if stdout is None:
|
||||
self.stdout = sys.stdout
|
||||
else:
|
||||
self.stdout = stdout
|
||||
|
||||
option_list = BaseCommand.option_list + (
|
||||
make_option('--all',
|
||||
action='store_true',
|
||||
dest='all',
|
||||
default=False,
|
||||
help='Scan all packages'),
|
||||
make_option('--purge-packages',
|
||||
action='store_true',
|
||||
dest='purge-packages',
|
||||
default=False,
|
||||
help='Purge old packages'),
|
||||
make_option('--purge-versions',
|
||||
action='store_true',
|
||||
dest='purge-versions',
|
||||
default=False,
|
||||
help='Purge old versions'),
|
||||
make_option('--no-log',
|
||||
action='store_true',
|
||||
dest='no-log',
|
||||
default=False,
|
||||
help='Don\'t store logs'),
|
||||
make_option('--prefetch',
|
||||
action='store_true',
|
||||
dest='prefetch',
|
||||
default=False,
|
||||
help=('Prefetch all versions and packages from DB to '
|
||||
'speedup full scan process.')),
|
||||
make_option('--quiet',
|
||||
action='store_true',
|
||||
dest='quiet',
|
||||
default=False,
|
||||
help='Be quiet'),
|
||||
)
|
||||
args = '[package package ...]'
|
||||
help = 'Scans portage tree and fills database'
|
||||
|
||||
_cache = {'packages': {}, 'versions': {}}
|
||||
self.options = options
|
||||
self._cache = {'packages': {}, 'versions': {}}
|
||||
self._overlays = None
|
||||
|
||||
def cache_hash_package(self, category, name):
|
||||
return '%s/%s' % (category, name)
|
||||
@ -84,38 +56,6 @@ class Command(BaseCommand):
|
||||
)
|
||||
self._cache['versions'][key] = version
|
||||
|
||||
def handle(self, *args, **options):
|
||||
self.options = options
|
||||
|
||||
if not options['quiet']:
|
||||
self.stdout.write('Scanning portage tree...\n')
|
||||
|
||||
if options['prefetch']:
|
||||
if not options['quiet']:
|
||||
self.stdout.write('Prefetching objects...')
|
||||
self.stdout.flush()
|
||||
for package in Package.objects.all():
|
||||
self.cache_store_package(package)
|
||||
for version in Version.objects.select_related('package').all():
|
||||
self.cache_store_version(version)
|
||||
if not options['quiet']:
|
||||
self.stdout.write('done\n')
|
||||
|
||||
if options['all']:
|
||||
self.scan()
|
||||
elif len(args):
|
||||
for package in args:
|
||||
self.scan(package)
|
||||
else:
|
||||
for package in sys.stdin.readlines():
|
||||
self.scan(package[:-1])
|
||||
|
||||
if options['purge-versions']:
|
||||
self.purge_versions(options)
|
||||
|
||||
if not options['quiet']:
|
||||
self.stdout.write('Done.\n')
|
||||
|
||||
def overlays(self):
|
||||
if self._overlays:
|
||||
return self._overlays
|
||||
@ -143,7 +83,7 @@ class Command(BaseCommand):
|
||||
return self._overlays
|
||||
|
||||
@commit_on_success
|
||||
def scan(self, query=None):
|
||||
def run(self, query=None):
|
||||
env = os.environ
|
||||
env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
|
||||
|
||||
@ -177,7 +117,7 @@ class Command(BaseCommand):
|
||||
Package.objects.filter(name=query).delete()
|
||||
else:
|
||||
sys.stderr.write(
|
||||
self.style.ERROR(
|
||||
color_style.ERROR(
|
||||
"Unknown package '%s'\n" % query
|
||||
)
|
||||
)
|
||||
@ -291,17 +231,19 @@ class Command(BaseCommand):
|
||||
if self.options['no-log']:
|
||||
return
|
||||
|
||||
entry = VersionLog.objects.create(package=obj.package,
|
||||
action=VersionLog.VERSION_ADDED)
|
||||
entry.slot = obj.slot
|
||||
entry.revision = obj.revision
|
||||
entry.version = obj.version
|
||||
entry.overlay = obj.overlay
|
||||
entry.save()
|
||||
VersionLog.objects.create(
|
||||
package=obj.package,
|
||||
action=VersionLog.VERSION_ADDED,
|
||||
slot=obj.slot,
|
||||
revision=obj.revision,
|
||||
version=obj.version,
|
||||
overlay=obj.overlay
|
||||
)
|
||||
|
||||
@commit_on_success
|
||||
def purge_versions(self, options):
|
||||
' For each dead versions '
|
||||
|
||||
@commit_on_success
|
||||
def purge_versions(options):
|
||||
# For each dead versions
|
||||
for version in Version.objects.filter(packaged=True, alive=False):
|
||||
if version.overlay == 'gentoo':
|
||||
version.package.n_packaged -= 1
|
||||
@ -310,10 +252,10 @@ class Command(BaseCommand):
|
||||
version.package.n_versions -= 1
|
||||
version.package.save()
|
||||
|
||||
if not self.options['quiet']:
|
||||
if not options['quiet']:
|
||||
sys.stdout.write('- [v] %s\n' % (version))
|
||||
|
||||
if self.options['no-log']:
|
||||
if options['no-log']:
|
||||
continue
|
||||
|
||||
entry = VersionLog.objects.create(
|
||||
@ -327,3 +269,75 @@ class Command(BaseCommand):
|
||||
entry.save()
|
||||
|
||||
Version.objects.filter(packaged=True, alive=False).delete()
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
|
||||
option_list = BaseCommand.option_list + (
|
||||
make_option('--all',
|
||||
action='store_true',
|
||||
dest='all',
|
||||
default=False,
|
||||
help='Scan all packages'),
|
||||
make_option('--purge-packages',
|
||||
action='store_true',
|
||||
dest='purge-packages',
|
||||
default=False,
|
||||
help='Purge old packages'),
|
||||
make_option('--purge-versions',
|
||||
action='store_true',
|
||||
dest='purge-versions',
|
||||
default=False,
|
||||
help='Purge old versions'),
|
||||
make_option('--no-log',
|
||||
action='store_true',
|
||||
dest='no-log',
|
||||
default=False,
|
||||
help='Don\'t store logs'),
|
||||
make_option('--prefetch',
|
||||
action='store_true',
|
||||
dest='prefetch',
|
||||
default=False,
|
||||
help=('Prefetch all versions and packages from DB to '
|
||||
'speedup full scan process.')),
|
||||
make_option('--quiet',
|
||||
action='store_true',
|
||||
dest='quiet',
|
||||
default=False,
|
||||
help='Be quiet'),
|
||||
)
|
||||
args = '[package package ...]'
|
||||
help = 'Scans portage tree and fills database'
|
||||
|
||||
def handle(self, *args, **options):
|
||||
scan_portage = ScanPortage(stdout=self.stdout, **options)
|
||||
|
||||
if not options['quiet']:
|
||||
self.stdout.write('Scanning portage tree...\n')
|
||||
|
||||
if options['prefetch']:
|
||||
if not options['quiet']:
|
||||
self.stdout.write('Prefetching objects...')
|
||||
self.stdout.flush()
|
||||
for package in Package.objects.all():
|
||||
scan_portage.cache_store_package(package)
|
||||
for version in Version.objects.select_related('package').all():
|
||||
scan_portage.cache_store_version(version)
|
||||
if not options['quiet']:
|
||||
self.stdout.write('done\n')
|
||||
|
||||
if options['all']:
|
||||
scan_portage.run()
|
||||
elif len(args):
|
||||
for package in args:
|
||||
scan_portage.run(package)
|
||||
else:
|
||||
for package in sys.stdin.readlines():
|
||||
scan_portage.run(package[:-1])
|
||||
|
||||
if options['purge-versions']:
|
||||
purge_versions(options)
|
||||
|
||||
if not options['quiet']:
|
||||
self.stdout.write('Done.\n')
|
||||
@ -2,16 +2,150 @@ import subprocess
|
||||
import portage
|
||||
import sys
|
||||
import re
|
||||
|
||||
from StringIO import StringIO
|
||||
from optparse import make_option
|
||||
from collections import defaultdict
|
||||
|
||||
from django.utils import timezone
|
||||
from django.db.transaction import commit_on_success
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from djeuscan.models import Package, Version, EuscanResult, VersionLog
|
||||
|
||||
|
||||
class ScanUpstream(object):
|
||||
def __init__(self, options=None):
|
||||
if options is None:
|
||||
self.options = defaultdict(None)
|
||||
else:
|
||||
self.options = options
|
||||
|
||||
def run(self, packages=None):
|
||||
for package in packages:
|
||||
cmd = ['euscan', package]
|
||||
|
||||
fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
output = StringIO(fp.communicate()[0])
|
||||
|
||||
self.parse_output(output)
|
||||
|
||||
def store_result(self, package, log):
|
||||
# Remove previous logs
|
||||
EuscanResult.objects.filter(package=package).delete()
|
||||
|
||||
obj = EuscanResult()
|
||||
obj.package = package
|
||||
obj.result = log
|
||||
obj.datetime = timezone.now()
|
||||
obj.save()
|
||||
|
||||
def store_package(self, cpv):
|
||||
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
|
||||
|
||||
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
|
||||
|
||||
if created and not self.options['quiet']:
|
||||
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
|
||||
|
||||
# Set all versions dead, then set found versions alive and
|
||||
# delete old versions
|
||||
Version.objects.filter(package=obj, packaged=False).update(alive=False)
|
||||
|
||||
return obj
|
||||
|
||||
def store_version(self, package, ver, url):
|
||||
obj, created = Version.objects.get_or_create(
|
||||
package=package, slot='', revision='r0', version=ver, overlay=''
|
||||
)
|
||||
|
||||
obj.alive = True
|
||||
obj.urls = url
|
||||
obj.packaged = False
|
||||
obj.save()
|
||||
|
||||
# If it's not a new version, just update the object and continue
|
||||
if not created:
|
||||
return
|
||||
|
||||
if not self.options['quiet']:
|
||||
sys.stdout.write('+ [u] %s %s\n' % (obj, url))
|
||||
|
||||
entry = VersionLog.objects.create(
|
||||
package=package,
|
||||
action=VersionLog.VERSION_ADDED
|
||||
)
|
||||
entry.slot = ''
|
||||
entry.revision = 'r0'
|
||||
entry.version = ver
|
||||
entry.overlay = ''
|
||||
entry.save()
|
||||
|
||||
package.n_versions += 1
|
||||
package.save()
|
||||
|
||||
@commit_on_success
|
||||
def parse_output(self, output):
|
||||
from portage.versions import _cp
|
||||
|
||||
package_re = re.compile(
|
||||
r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$'
|
||||
)
|
||||
version_re = re.compile(
|
||||
r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$'
|
||||
)
|
||||
|
||||
package = None
|
||||
log = ""
|
||||
|
||||
while True:
|
||||
line = output.readline()
|
||||
if line == '':
|
||||
break
|
||||
match = package_re.match(line)
|
||||
if match:
|
||||
if package:
|
||||
self.store_result(package, log)
|
||||
|
||||
cpv = match.group('cpv')
|
||||
package = self.store_package(cpv)
|
||||
log = line
|
||||
continue
|
||||
|
||||
log += line
|
||||
|
||||
match = version_re.match(line)
|
||||
if match:
|
||||
ver = match.group('ver')
|
||||
url = match.group('url')
|
||||
self.store_version(package, ver, url)
|
||||
|
||||
if package:
|
||||
self.store_result(package, log)
|
||||
|
||||
|
||||
@commit_on_success
|
||||
def purge_versions(options):
|
||||
# For each dead versions
|
||||
for version in Version.objects.filter(packaged=False, alive=False):
|
||||
entry = VersionLog.objects.create(
|
||||
package=version.package,
|
||||
action=VersionLog.VERSION_REMOVED
|
||||
)
|
||||
entry.slot = version.slot
|
||||
entry.revision = version.revision
|
||||
entry.version = version.version
|
||||
entry.overlay = version.overlay
|
||||
entry.save()
|
||||
|
||||
version.package.n_versions -= 1
|
||||
version.package.save()
|
||||
|
||||
if not options['quiet']:
|
||||
sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
|
||||
Version.objects.filter(packaged=False, alive=False).delete()
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
|
||||
@ -41,10 +175,12 @@ class Command(BaseCommand):
|
||||
help = 'Scans metadata and fills database'
|
||||
|
||||
def handle(self, *args, **options):
|
||||
scan_upstream = ScanUpstream(options)
|
||||
|
||||
if options['feed']:
|
||||
self.parse_output(options, sys.stdin)
|
||||
scan_upstream.parse_output(options, sys.stdin)
|
||||
if options['purge-versions']:
|
||||
self.purge_versions(options)
|
||||
purge_versions(options)
|
||||
return
|
||||
|
||||
if not options['quiet']:
|
||||
@ -60,133 +196,10 @@ class Command(BaseCommand):
|
||||
else:
|
||||
packages = [package[:-1] for package in sys.stdin.readlines()]
|
||||
|
||||
self.scan(options, packages)
|
||||
scan_upstream.run(packages)
|
||||
|
||||
if options['purge-versions']:
|
||||
self.purge_versions(options)
|
||||
purge_versions(options)
|
||||
|
||||
if not options['quiet']:
|
||||
self.stdout.write('Done.\n')
|
||||
|
||||
def scan(self, options, packages=None):
|
||||
for package in packages:
|
||||
cmd = ['euscan', package]
|
||||
|
||||
fp = subprocess.Popen(cmd, stdout=subprocess.PIPE)
|
||||
output = StringIO(fp.communicate()[0])
|
||||
|
||||
self.parse_output(options, output)
|
||||
|
||||
@commit_on_success
|
||||
def parse_output(self, options, output):
|
||||
from portage.versions import _cp
|
||||
|
||||
package_re = re.compile(
|
||||
r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$'
|
||||
)
|
||||
version_re = re.compile(
|
||||
r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$'
|
||||
)
|
||||
|
||||
package = None
|
||||
log = ""
|
||||
|
||||
while True:
|
||||
line = output.readline()
|
||||
if line == '':
|
||||
break
|
||||
match = package_re.match(line)
|
||||
if match:
|
||||
if package:
|
||||
self.store_result(options, package, log)
|
||||
|
||||
cpv = match.group('cpv')
|
||||
package = self.store_package(options, cpv)
|
||||
log = line
|
||||
continue
|
||||
|
||||
log += line
|
||||
|
||||
match = version_re.match(line)
|
||||
if match:
|
||||
ver = match.group('ver')
|
||||
url = match.group('url')
|
||||
self.store_version(options, package, ver, url)
|
||||
|
||||
if package:
|
||||
self.store_result(options, package, log)
|
||||
|
||||
def store_result(self, options, package, log):
|
||||
# Remove previous logs
|
||||
EuscanResult.objects.filter(package=package).delete()
|
||||
|
||||
obj = EuscanResult()
|
||||
obj.package = package
|
||||
obj.result = log
|
||||
obj.datetime = timezone.now()
|
||||
obj.save()
|
||||
|
||||
def store_package(self, options, cpv):
|
||||
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
|
||||
|
||||
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
|
||||
|
||||
if created and not options['quiet']:
|
||||
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
|
||||
|
||||
# Set all versions dead, then set found versions alive and
|
||||
# delete old versions
|
||||
Version.objects.filter(package=obj, packaged=False).update(alive=False)
|
||||
|
||||
return obj
|
||||
|
||||
def store_version(self, options, package, ver, url):
|
||||
obj, created = Version.objects.get_or_create(
|
||||
package=package, slot='', revision='r0', version=ver, overlay=''
|
||||
)
|
||||
|
||||
obj.alive = True
|
||||
obj.urls = url
|
||||
obj.packaged = False
|
||||
obj.save()
|
||||
|
||||
''' If it's not a new version, just update the object and continue '''
|
||||
if not created:
|
||||
return
|
||||
|
||||
if not options['quiet']:
|
||||
sys.stdout.write('+ [u] %s %s\n' % (obj, url))
|
||||
|
||||
entry = VersionLog.objects.create(
|
||||
package=package,
|
||||
action=VersionLog.VERSION_ADDED
|
||||
)
|
||||
entry.slot = ''
|
||||
entry.revision = 'r0'
|
||||
entry.version = ver
|
||||
entry.overlay = ''
|
||||
entry.save()
|
||||
|
||||
package.n_versions += 1
|
||||
package.save()
|
||||
|
||||
@commit_on_success
|
||||
def purge_versions(self, options):
|
||||
' For each dead versions '
|
||||
for version in Version.objects.filter(packaged=False, alive=False):
|
||||
entry = VersionLog.objects.create(
|
||||
package=version.package,
|
||||
action=VersionLog.VERSION_REMOVED
|
||||
)
|
||||
entry.slot = version.slot
|
||||
entry.revision = version.revision
|
||||
entry.version = version.version
|
||||
entry.overlay = version.overlay
|
||||
entry.save()
|
||||
|
||||
version.package.n_versions -= 1
|
||||
version.package.save()
|
||||
|
||||
if not options['quiet']:
|
||||
sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
|
||||
Version.objects.filter(packaged=False, alive=False).delete()
|
||||
@ -1,199 +0,0 @@
|
||||
from optparse import make_option
|
||||
|
||||
from django.db.transaction import commit_on_success
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils import timezone
|
||||
|
||||
from djeuscan.models import Package, Herd, Maintainer, Version
|
||||
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
|
||||
from djeuscan import charts
|
||||
|
||||
from distutils.version import StrictVersion, LooseVersion
|
||||
|
||||
|
||||
def compare_versions(version1, version2):
|
||||
try:
|
||||
return cmp(StrictVersion(version1), StrictVersion(version2))
|
||||
# in case of abnormal version number, fall back to LooseVersion
|
||||
except ValueError:
|
||||
return cmp(LooseVersion(version1), LooseVersion(version2))
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
help = 'Update counters'
|
||||
|
||||
option_list = BaseCommand.option_list + (
|
||||
make_option('--quiet',
|
||||
action='store_true',
|
||||
dest='quiet',
|
||||
default=False,
|
||||
help='Be quiet'),
|
||||
make_option('--fast',
|
||||
action='store_true',
|
||||
dest='fast',
|
||||
default=False,
|
||||
help='Skip sanity checks'),
|
||||
make_option('--nolog',
|
||||
action='store_true',
|
||||
dest='nolog',
|
||||
default=False,
|
||||
help='Skip logs'),
|
||||
|
||||
)
|
||||
|
||||
@commit_on_success
|
||||
def handle(self, *args, **options):
|
||||
now = timezone.now()
|
||||
|
||||
categories = {}
|
||||
herds = {}
|
||||
maintainers = {}
|
||||
|
||||
wlog = None
|
||||
|
||||
if not options['nolog']:
|
||||
wlog = WorldLog()
|
||||
wlog.datetime = now
|
||||
|
||||
for cat in Package.objects.values('category').distinct():
|
||||
clog = CategoryLog()
|
||||
clog.datetime = now
|
||||
clog.category = cat['category']
|
||||
categories[clog.category] = clog
|
||||
|
||||
for herd in Herd.objects.all():
|
||||
hlog = HerdLog()
|
||||
hlog.datetime = now
|
||||
hlog.herd = herd
|
||||
herds[herd.id] = hlog
|
||||
|
||||
for maintainer in Maintainer.objects.all():
|
||||
mlog = MaintainerLog()
|
||||
mlog.datetime = now
|
||||
mlog.maintainer = maintainer
|
||||
maintainers[maintainer.id] = mlog
|
||||
|
||||
package_queryset = Package.objects.all()
|
||||
|
||||
n_versions = {}
|
||||
n_packaged = {}
|
||||
n_overlay = {}
|
||||
|
||||
last_versions_gentoo = {}
|
||||
last_versions_overlay = {}
|
||||
last_versions_upstream = {}
|
||||
|
||||
def add_safe(storage, key):
|
||||
if key not in storage:
|
||||
storage[key] = 1
|
||||
else:
|
||||
storage[key] += 1
|
||||
|
||||
def add_last_ver(storage, version):
|
||||
key = version['package_id']
|
||||
if key not in storage:
|
||||
storage[key] = version
|
||||
return
|
||||
if version['version'].startswith('9999'):
|
||||
return
|
||||
if compare_versions(storage[key]['version'],
|
||||
version['version']) < 0:
|
||||
storage[key] = version
|
||||
|
||||
if not options['fast']:
|
||||
attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
|
||||
for version in Version.objects.all().values(*attrs):
|
||||
overlay, packaged = version['overlay'], version['packaged']
|
||||
package_id = version['package_id']
|
||||
|
||||
add_safe(n_versions, package_id)
|
||||
|
||||
if not packaged:
|
||||
add_last_ver(last_versions_upstream, version)
|
||||
continue
|
||||
if overlay == 'gentoo':
|
||||
add_safe(n_packaged, package_id)
|
||||
add_last_ver(last_versions_gentoo, version)
|
||||
else:
|
||||
add_safe(n_overlay, package_id)
|
||||
add_last_ver(last_versions_overlay, version)
|
||||
|
||||
for package in package_queryset.select_related('herds', 'maintainers'):
|
||||
if not options['fast']:
|
||||
package.n_versions = n_versions.get(package.id, 0)
|
||||
package.n_packaged = n_packaged.get(package.id, 0)
|
||||
package.n_overlay = n_overlay.get(package.id, 0)
|
||||
|
||||
default = {'id': None}
|
||||
package.last_version_gentoo_id = last_versions_gentoo.get(
|
||||
package.id, default
|
||||
)['id']
|
||||
package.last_version_overlay_id = last_versions_overlay.get(
|
||||
package.id, default
|
||||
)['id']
|
||||
package.last_version_upstream_id = last_versions_upstream.get(
|
||||
package.id, default
|
||||
)['id']
|
||||
|
||||
package.save()
|
||||
|
||||
n_packages_gentoo = int(package.n_packaged == package.n_versions)
|
||||
n_packages_overlay = int(package.n_overlay and package.n_packaged \
|
||||
+ package.n_overlay == package.n_versions)
|
||||
n_packages_outdated = int(package.n_packaged + package.n_overlay \
|
||||
< package.n_versions)
|
||||
|
||||
def update_row(storage, key):
|
||||
storage[key].n_packages_gentoo += n_packages_gentoo
|
||||
storage[key].n_packages_overlay += n_packages_overlay
|
||||
storage[key].n_packages_outdated += n_packages_outdated
|
||||
|
||||
storage[key].n_versions_gentoo += package.n_packaged
|
||||
storage[key].n_versions_overlay += package.n_overlay
|
||||
storage[key].n_versions_upstream += package.n_versions - \
|
||||
package.n_packaged - \
|
||||
package.n_overlay
|
||||
|
||||
def update_log(storage, qs):
|
||||
for row in qs:
|
||||
update_row(storage, row['id'])
|
||||
|
||||
if not options['nolog']:
|
||||
update_log(herds, package.herds.all().values('id'))
|
||||
update_log(maintainers, package.maintainers.all().values('id'))
|
||||
update_row(categories, package.category)
|
||||
|
||||
wlog.n_packages_gentoo += n_packages_gentoo
|
||||
wlog.n_packages_overlay += n_packages_overlay
|
||||
wlog.n_packages_outdated += n_packages_outdated
|
||||
|
||||
wlog.n_versions_gentoo += package.n_packaged
|
||||
wlog.n_versions_overlay += package.n_overlay
|
||||
wlog.n_versions_upstream += package.n_versions - \
|
||||
package.n_packaged - \
|
||||
package.n_overlay
|
||||
|
||||
if options['nolog']:
|
||||
return
|
||||
|
||||
for clog in categories.values():
|
||||
if not options['quiet']:
|
||||
self.stdout.write('+ [cl] %s\n' % clog)
|
||||
charts.rrd_update('category-%s' % clog.category, now, clog)
|
||||
clog.save()
|
||||
|
||||
for hlog in herds.values():
|
||||
if not options['quiet']:
|
||||
self.stdout.write('+ [hl] %s\n' % hlog)
|
||||
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
|
||||
hlog.save()
|
||||
|
||||
for mlog in maintainers.values():
|
||||
if not options['quiet']:
|
||||
self.stdout.write('+ [ml] %s\n' % mlog)
|
||||
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
|
||||
mlog.save()
|
||||
|
||||
charts.rrd_update('world', now, wlog)
|
||||
wlog.save()
|
||||
208
euscanwww/djeuscan/management/commands/update_counters.py
Normal file
208
euscanwww/djeuscan/management/commands/update_counters.py
Normal file
@ -0,0 +1,208 @@
|
||||
import sys
|
||||
from optparse import make_option
|
||||
|
||||
from django.db.transaction import commit_on_success
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils import timezone
|
||||
|
||||
from djeuscan.models import Package, Herd, Maintainer, Version
|
||||
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
|
||||
from djeuscan import charts
|
||||
|
||||
from distutils.version import StrictVersion, LooseVersion
|
||||
|
||||
|
||||
def compare_versions(version1, version2):
|
||||
try:
|
||||
return cmp(StrictVersion(version1), StrictVersion(version2))
|
||||
# in case of abnormal version number, fall back to LooseVersion
|
||||
except ValueError:
|
||||
return cmp(LooseVersion(version1), LooseVersion(version2))
|
||||
|
||||
|
||||
def add_safe(storage, key):
|
||||
if key not in storage:
|
||||
storage[key] = 1
|
||||
else:
|
||||
storage[key] += 1
|
||||
|
||||
|
||||
def add_last_ver(storage, version):
|
||||
key = version['package_id']
|
||||
if key not in storage:
|
||||
storage[key] = version
|
||||
return
|
||||
if version['version'].startswith('9999'):
|
||||
return
|
||||
if compare_versions(storage[key]['version'],
|
||||
version['version']) < 0:
|
||||
storage[key] = version
|
||||
|
||||
|
||||
@commit_on_success
|
||||
def update_counters(stdout=None, **options):
|
||||
if stdout is None:
|
||||
stdout = sys.stdout
|
||||
|
||||
now = timezone.now()
|
||||
|
||||
categories = {}
|
||||
herds = {}
|
||||
maintainers = {}
|
||||
|
||||
wlog = None
|
||||
|
||||
if not options['nolog']:
|
||||
wlog = WorldLog()
|
||||
wlog.datetime = now
|
||||
|
||||
for cat in Package.objects.values('category').distinct():
|
||||
clog = CategoryLog()
|
||||
clog.datetime = now
|
||||
clog.category = cat['category']
|
||||
categories[clog.category] = clog
|
||||
|
||||
for herd in Herd.objects.all():
|
||||
hlog = HerdLog()
|
||||
hlog.datetime = now
|
||||
hlog.herd = herd
|
||||
herds[herd.id] = hlog
|
||||
|
||||
for maintainer in Maintainer.objects.all():
|
||||
mlog = MaintainerLog()
|
||||
mlog.datetime = now
|
||||
mlog.maintainer = maintainer
|
||||
maintainers[maintainer.id] = mlog
|
||||
|
||||
package_queryset = Package.objects.all()
|
||||
|
||||
n_versions = {}
|
||||
n_packaged = {}
|
||||
n_overlay = {}
|
||||
|
||||
last_versions_gentoo = {}
|
||||
last_versions_overlay = {}
|
||||
last_versions_upstream = {}
|
||||
|
||||
if not options['fast']:
|
||||
attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
|
||||
for version in Version.objects.all().values(*attrs):
|
||||
overlay, packaged = version['overlay'], version['packaged']
|
||||
package_id = version['package_id']
|
||||
|
||||
add_safe(n_versions, package_id)
|
||||
|
||||
if not packaged:
|
||||
add_last_ver(last_versions_upstream, version)
|
||||
continue
|
||||
if overlay == 'gentoo':
|
||||
add_safe(n_packaged, package_id)
|
||||
add_last_ver(last_versions_gentoo, version)
|
||||
else:
|
||||
add_safe(n_overlay, package_id)
|
||||
add_last_ver(last_versions_overlay, version)
|
||||
|
||||
for package in package_queryset.select_related('herds', 'maintainers'):
|
||||
if not options['fast']:
|
||||
package.n_versions = n_versions.get(package.id, 0)
|
||||
package.n_packaged = n_packaged.get(package.id, 0)
|
||||
package.n_overlay = n_overlay.get(package.id, 0)
|
||||
|
||||
default = {'id': None}
|
||||
package.last_version_gentoo_id = last_versions_gentoo.get(
|
||||
package.id, default
|
||||
)['id']
|
||||
package.last_version_overlay_id = last_versions_overlay.get(
|
||||
package.id, default
|
||||
)['id']
|
||||
package.last_version_upstream_id = last_versions_upstream.get(
|
||||
package.id, default
|
||||
)['id']
|
||||
|
||||
package.save()
|
||||
|
||||
n_packages_gentoo = int(package.n_packaged == package.n_versions)
|
||||
n_packages_overlay = int(package.n_overlay and package.n_packaged \
|
||||
+ package.n_overlay == package.n_versions)
|
||||
n_packages_outdated = int(package.n_packaged + package.n_overlay \
|
||||
< package.n_versions)
|
||||
|
||||
def update_row(storage, key):
|
||||
storage[key].n_packages_gentoo += n_packages_gentoo
|
||||
storage[key].n_packages_overlay += n_packages_overlay
|
||||
storage[key].n_packages_outdated += n_packages_outdated
|
||||
|
||||
storage[key].n_versions_gentoo += package.n_packaged
|
||||
storage[key].n_versions_overlay += package.n_overlay
|
||||
storage[key].n_versions_upstream += package.n_versions - \
|
||||
package.n_packaged - \
|
||||
package.n_overlay
|
||||
|
||||
def update_log(storage, qs):
|
||||
for row in qs:
|
||||
update_row(storage, row['id'])
|
||||
|
||||
if not options['nolog']:
|
||||
update_log(herds, package.herds.all().values('id'))
|
||||
update_log(maintainers, package.maintainers.all().values('id'))
|
||||
update_row(categories, package.category)
|
||||
|
||||
wlog.n_packages_gentoo += n_packages_gentoo
|
||||
wlog.n_packages_overlay += n_packages_overlay
|
||||
wlog.n_packages_outdated += n_packages_outdated
|
||||
|
||||
wlog.n_versions_gentoo += package.n_packaged
|
||||
wlog.n_versions_overlay += package.n_overlay
|
||||
wlog.n_versions_upstream += package.n_versions - \
|
||||
package.n_packaged - \
|
||||
package.n_overlay
|
||||
|
||||
if options['nolog']:
|
||||
return
|
||||
|
||||
for clog in categories.values():
|
||||
if not options['quiet']:
|
||||
stdout.write('+ [cl] %s\n' % clog)
|
||||
charts.rrd_update('category-%s' % clog.category, now, clog)
|
||||
clog.save()
|
||||
|
||||
for hlog in herds.values():
|
||||
if not options['quiet']:
|
||||
stdout.write('+ [hl] %s\n' % hlog)
|
||||
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
|
||||
hlog.save()
|
||||
|
||||
for mlog in maintainers.values():
|
||||
if not options['quiet']:
|
||||
stdout.write('+ [ml] %s\n' % mlog)
|
||||
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
|
||||
mlog.save()
|
||||
|
||||
charts.rrd_update('world', now, wlog)
|
||||
wlog.save()
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
help = 'Update counters'
|
||||
|
||||
option_list = BaseCommand.option_list + (
|
||||
make_option('--quiet',
|
||||
action='store_true',
|
||||
dest='quiet',
|
||||
default=False,
|
||||
help='Be quiet'),
|
||||
make_option('--fast',
|
||||
action='store_true',
|
||||
dest='fast',
|
||||
default=False,
|
||||
help='Skip sanity checks'),
|
||||
make_option('--nolog',
|
||||
action='store_true',
|
||||
dest='nolog',
|
||||
default=False,
|
||||
help='Skip logs'),
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
update_counters(stdout=self.stdout, **options)
|
||||
5
setup.py
5
setup.py
@ -87,8 +87,9 @@ setup(
|
||||
'https://github.com/iksaif/euscan/tarball/' +
|
||||
('master' if __version__ == '9999' else ('euscan-%s' % __version__))
|
||||
),
|
||||
install_requires=['Django==1.4', 'django-annoying', 'South',
|
||||
'django-piston', 'BeautifulSoup', "matplotlib"],
|
||||
install_requires=['Django==1.4', 'django-annoying==0..7.6', 'South==0.7.4',
|
||||
'django-piston==0.2.3', 'BeautifulSoup==3.2.1',
|
||||
'matplotlib==1.1.0', 'django-celery==2.5.5'],
|
||||
package_dir={'': 'pym'},
|
||||
packages=packages,
|
||||
package_data={},
|
||||
|
||||
Loading…
Reference in New Issue
Block a user