src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Lovely day for PEP8 and pylint!

Browse Source
This commit is contained in:
volpino
2012-04-28 18:16:05 +02:00
parent 4e4f9643ac
commit 8c91855a58
37 changed files with 722 additions and 388 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
euscanwww/djeuscan/management/commands/list-packages.py
View File

@@ -1,8 +1,9 @@
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import Package
class Command(BaseCommand):
_overlays = {}
help = 'List packages'

13
euscanwww/djeuscan/management/commands/regen-rrds.py
View File

@@ -1,11 +1,8 @@
import datetime
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from djeuscan import charts
class Command(BaseCommand):
_overlays = {}
help = 'Regenerate rrd database'
@@ -15,10 +12,12 @@ class Command(BaseCommand):
charts.rrd_update('world', wlog.datetime, wlog)
for clog in CategoryLog.objects.all():
charts.rrd_update('category-%s' % clog.category, clog.datetime, clog)
charts.rrd_update('category-%s' % clog.category,
clog.datetime, clog)
for hlog in HerdLog.objects.all():
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
for mlog in MaintainerLog.objects.all():
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, mlog.datetime, mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
mlog.datetime, mlog)

53
euscanwww/djeuscan/management/commands/scan-metadata.py
View File

@@ -1,19 +1,15 @@
import subprocess
import portage
import sys
import os
import re
from portage import versions
from optparse import make_option
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import Package, Herd, Maintainer
from gentoolkit.query import Query
from gentoolkit.errors import GentoolkitFatalError
class Command(BaseCommand):
_overlays = {}
@@ -53,16 +49,20 @@ class Command(BaseCommand):
)
if not matches:
sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % query))
sys.stderr.write(
self.style.ERROR("Unknown package '%s'\n" % query)
)
return
matches = sorted(matches)
matches = sorted(matches)
pkg = matches.pop()
if '9999' in pkg.version and len(matches):
if '9999' in pkg.version and len(matches):
pkg = matches.pop()
if not obj:
obj, created = Package.objects.get_or_create(category=pkg.category, name=pkg.name)
obj, created = Package.objects.get_or_create(
category=pkg.category, name=pkg.name
)
else:
created = False
@@ -70,22 +70,34 @@ class Command(BaseCommand):
obj.homepage = pkg.environment("HOMEPAGE")
obj.description = pkg.environment("DESCRIPTION")
except GentoolkitFatalError, err:
sys.stderr.write(self.style.ERROR("Gentoolkit fatal error: '%s'\n" % str(err)))
sys.stderr.write(
self.style.ERROR(
"Gentoolkit fatal error: '%s'\n" % str(err)
)
)
if created and not self.options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
if pkg.metadata:
herds = dict([(herd[0], herd) for herd in pkg.metadata.herds(True)])
maintainers = dict([(m.email, m) for m in pkg.metadata.maintainers()])
herds = dict(
[(herd[0], herd) for herd in pkg.metadata.herds(True)]
)
maintainers = dict(
[(m.email, m) for m in pkg.metadata.maintainers()]
)
existing_herds = [h.herd for h in obj.herds.all()]
new_herds = set(herds.keys()).difference(existing_herds)
old_herds = set(existing_herds).difference(herds.keys())
existing_maintainers = [m.email for m in obj.maintainers.all()]
new_maintainers = set(maintainers.keys()).difference(existing_maintainers)
old_maintainers = set(existing_maintainers).difference(maintainers.keys())
new_maintainers = set(
maintainers.keys()).difference(existing_maintainers
)
old_maintainers = set(
existing_maintainers).difference(maintainers.keys()
)
for herd in obj.herds.all():
if herd.herd in old_herds:
@@ -101,7 +113,9 @@ class Command(BaseCommand):
for maintainer in new_maintainers:
maintainer = maintainers[maintainer]
maintainer = self.store_maintainer(maintainer.name, maintainer.email)
maintainer = self.store_maintainer(
maintainer.name, maintainer.email
)
obj.maintainers.add(maintainer)
obj.save()
@@ -131,9 +145,12 @@ class Command(BaseCommand):
if created:
if not self.options['quiet']:
sys.stdout.write('+ [m] %s <%s>\n' % (name.encode('utf-8'), email))
sys.stdout.write(
'+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
)
if not maintainer.name or name not in [maintainer.name, email, '{nil}']:
if not maintainer.name or \
name not in [maintainer.name, email, '{nil}']:
maintainer.name = name
maintainer.save()

94
euscanwww/djeuscan/management/commands/scan-portage.py
View File

@@ -4,13 +4,13 @@ import sys
import os
import re
from portage import versions
from optparse import make_option
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import Package, Version, VersionLog
class Command(BaseCommand):
_overlays = {}
@@ -39,7 +39,8 @@ class Command(BaseCommand):
action='store_true',
dest='prefetch',
default=False,
help='Prefetch all versions and packages from DB to speedup full scan process.'),
help=('Prefetch all versions and packages from DB to '
'speedup full scan process.')),
make_option('--quiet',
action='store_true',
dest='quiet',
@@ -49,7 +50,7 @@ class Command(BaseCommand):
args = '[package package ...]'
help = 'Scans portage tree and fills database'
_cache = {'packages' : {}, 'versions' : {}}
_cache = {'packages': {}, 'versions': {}}
def cache_hash_package(self, category, name):
return '%s/%s' % (category, name)
@@ -59,22 +60,28 @@ class Command(BaseCommand):
self._cache['packages'][key] = package
def cache_get_package(self, category, name):
return self._cache['packages'].get(self.cache_hash_package(category, name))
return self._cache['packages'].get(
self.cache_hash_package(category, name)
)
def cache_hash_version(self, category, name, version, revision, slot, overlay):
def cache_hash_version(self, category, name, version, revision, slot,
overlay):
key = '%s/%s-%s-r%s %s %s' % (category, name,
version, revision,
slot, overlay)
return key
def cache_get_version(self, category, name, version, revision, slot, overlay):
key = self.cache_hash_version(category, name, version, revision, slot, overlay)
def cache_get_version(self, category, name, version, revision, slot,
overlay):
key = self.cache_hash_version(category, name, version, revision, slot,
overlay)
return self._cache['versions'].get(key)
def cache_store_version(self, version):
key = self.cache_hash_version(version.package.category, version.package.name,
version.version, version.revision, version.slot,
version.overlay)
key = self.cache_hash_version(
version.package.category, version.package.name, version.version,
version.revision, version.slot, version.overlay
)
self._cache['versions'][key] = version
def handle(self, *args, **options):
@@ -119,7 +126,8 @@ class Command(BaseCommand):
cmd = ['eix', '-!']
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
output = output.strip().strip('\n').split('\n')
overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
@@ -139,9 +147,10 @@ class Command(BaseCommand):
env = os.environ
env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages', '-x']
if query:
cmd.extend(['--exact', query])
cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages',
'-x']
if query:
cmd.extend(['--exact', query])
if self.options['all']:
if not self.options['quiet']:
@@ -151,7 +160,8 @@ class Command(BaseCommand):
if not self.options['quiet']:
self.stdout.write('done\n')
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
output = output.strip().strip('\n')
if len(output) == 0:
@@ -166,17 +176,23 @@ class Command(BaseCommand):
else:
Package.objects.filter(name=query).delete()
else:
sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % query))
sys.stderr.write(
self.style.ERROR(
"Unknown package '%s'\n" % query
)
)
return
output = output.split('\n')
output = output.split('\n')
packages = {}
line_re = re.compile(r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$')
line_re = re.compile(
r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$'
)
package = None
for line in output:
for line in output:
match = line_re.match(line)
if not match:
@@ -190,7 +206,8 @@ class Command(BaseCommand):
packages['%s/%s' % (cat, pkg)] = True
if not package or not (cat == package.category and pkg == package.name):
if not package or not \
(cat == package.category and pkg == package.name):
package = self.store_package(cat, pkg)
self.store_version(package, cpv, slot, overlay)
@@ -208,16 +225,23 @@ class Command(BaseCommand):
obj = self.cache_get_package(cat, pkg)
if not obj:
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
obj, created = Package.objects.get_or_create(
category=cat,
name=pkg
)
self.cache_store_package(obj)
if created:
if not self.options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
' Set all versions dead, then set found versions alive and delete old versions '
# Set all versions dead, then set found versions alive and
# delete old versions
if not self.options['all']:
Version.objects.filter(package=obj, packaged=True).update(alive=False)
Version.objects.filter(
package=obj,
packaged=True
).update(alive=False)
return obj
@@ -232,11 +256,15 @@ class Command(BaseCommand):
overlay = 'gentoo'
created = False
obj = self.cache_get_version(package.category, package.name, ver, rev, slot, overlay)
obj = self.cache_get_version(
package.category, package.name, ver, rev, slot, overlay
)
if not obj:
obj, created = Version.objects.get_or_create(package=package, slot=slot,
revision=rev, version=ver,
overlay=overlay)
obj, created = Version.objects.get_or_create(
package=package, slot=slot,
revision=rev, version=ver,
overlay=overlay
)
obj.alive = True
obj.packaged = True
@@ -245,7 +273,8 @@ class Command(BaseCommand):
if created:
self.cache_store_version(obj)
''' nothing to do (note: it can't be an upstream version because overlay can't be empty here) '''
# nothing to do (note: it can't be an upstream version because
# overlay can't be empty here)
if not created:
return
@@ -287,8 +316,10 @@ class Command(BaseCommand):
if self.options['no-log']:
continue
entry = VersionLog.objects.create(package=version.package,
action=VersionLog.VERSION_REMOVED)
entry = VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED
)
entry.slot = version.slot
entry.revision = version.revision
entry.version = version.version
@@ -296,4 +327,3 @@ class Command(BaseCommand):
entry.save()
Version.objects.filter(packaged=True, alive=False).delete()

38
euscanwww/djeuscan/management/commands/scan-upstream.py
View File

@@ -1,19 +1,17 @@
import subprocess
import portage
import sys
import os
import re
from StringIO import StringIO
from datetime import datetime
from portage import versions
from optparse import make_option
from django.utils import timezone
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import Package, Version, EuscanResult, VersionLog
class Command(BaseCommand):
_overlays = {}
@@ -60,7 +58,7 @@ class Command(BaseCommand):
elif args:
packages = list(args)
else:
packages = [ package[:-1] for package in sys.stdin.readlines() ]
packages = [package[:-1] for package in sys.stdin.readlines()]
self.scan(options, packages)
@@ -83,8 +81,12 @@ class Command(BaseCommand):
def parse_output(self, options, output):
from portage.versions import _cp
package_re = re.compile(r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$')
version_re = re.compile(r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$')
package_re = re.compile(
r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$'
)
version_re = re.compile(
r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$'
)
package = None
log = ""
@@ -124,7 +126,6 @@ class Command(BaseCommand):
obj.datetime = timezone.now()
obj.save()
def store_package(self, options, cpv):
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
@@ -133,15 +134,16 @@ class Command(BaseCommand):
if created and not options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
' Set all versions dead, then set found versions alive and delete old versions '
# Set all versions dead, then set found versions alive and
# delete old versions
Version.objects.filter(package=obj, packaged=False).update(alive=False)
return obj
def store_version(self, options, package, ver, url):
obj, created = Version.objects.get_or_create(package=package, slot='',
revision='r0', version=ver,
overlay='')
obj, created = Version.objects.get_or_create(
package=package, slot='', revision='r0', version=ver, overlay=''
)
obj.alive = True
obj.urls = url
@@ -155,7 +157,10 @@ class Command(BaseCommand):
if not options['quiet']:
sys.stdout.write('+ [u] %s %s\n' % (obj, url))
entry = VersionLog.objects.create(package=package, action=VersionLog.VERSION_ADDED)
entry = VersionLog.objects.create(
package=package,
action=VersionLog.VERSION_ADDED
)
entry.slot = ''
entry.revision = 'r0'
entry.version = ver
@@ -165,12 +170,14 @@ class Command(BaseCommand):
package.n_versions += 1
package.save()
@commit_on_success
def purge_versions(self, options):
' For each dead versions '
for version in Version.objects.filter(packaged=False, alive=False):
entry = VersionLog.objects.create(package=version.package, action=VersionLog.VERSION_REMOVED)
entry = VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED
)
entry.slot = version.slot
entry.revision = version.revision
entry.version = version.version
@@ -183,4 +190,3 @@ class Command(BaseCommand):
if not options['quiet']:
sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
Version.objects.filter(packaged=False, alive=False).delete()

55
euscanwww/djeuscan/management/commands/update-counters.py
View File

@@ -1,10 +1,7 @@
import datetime
from optparse import make_option
from django.db.models import Count, Sum
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from django.utils import timezone
from djeuscan.models import Package, Herd, Maintainer, Version
@@ -13,6 +10,7 @@ from djeuscan import charts
from distutils.version import StrictVersion, LooseVersion
def compare_versions(version1, version2):
try:
return cmp(StrictVersion(version1), StrictVersion(version2))
@@ -20,6 +18,7 @@ def compare_versions(version1, version2):
except ValueError:
return cmp(LooseVersion(version1), LooseVersion(version2))
class Command(BaseCommand):
_overlays = {}
help = 'Update counters'
@@ -98,7 +97,8 @@ class Command(BaseCommand):
return
if version['version'].startswith('9999'):
return
if compare_versions(storage[key]['version'], version['version']) < 0:
if compare_versions(storage[key]['version'],
version['version']) < 0:
storage[key] = version
if not options['fast']:
@@ -125,25 +125,36 @@ class Command(BaseCommand):
package.n_packaged = n_packaged.get(package.id, 0)
package.n_overlay = n_overlay.get(package.id, 0)
default = {'id' : None}
package.last_version_gentoo_id = last_versions_gentoo.get(package.id, default)['id']
package.last_version_overlay_id = last_versions_overlay.get(package.id, default)['id']
package.last_version_upstream_id = last_versions_upstream.get(package.id, default)['id']
default = {'id': None}
package.last_version_gentoo_id = last_versions_gentoo.get(
package.id, default
)['id']
package.last_version_overlay_id = last_versions_overlay.get(
package.id, default
)['id']
package.last_version_upstream_id = last_versions_upstream.get(
package.id, default
)['id']
package.save()
n_packages_gentoo = int(package.n_packaged == package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged + package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay < package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged \
+ package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay \
< package.n_versions)
def update_row(storage, key):
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_outdated += n_packages_outdated
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
def update_log(storage, qs):
for row in qs:
update_row(storage, row['id'])
@@ -153,13 +164,15 @@ class Command(BaseCommand):
update_log(maintainers, package.maintainers.all().values('id'))
update_row(categories, package.category)
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
if options['nolog']:
return