Merge branch 'master' of git://github.com/volpino/euscan

Conflicts:
	euscanwww/djeuscan/management/commands/scan_upstream.py

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>
This commit is contained in:
Corentin Chary 2012-06-30 07:38:26 +02:00
commit 20f796df60
84 changed files with 3340 additions and 1237 deletions

View File

@ -21,8 +21,8 @@ __description__ = "A tool to detect new upstream releases."
import sys import sys
import os import os
import getopt import getopt
import errno from errno import EINTR, EINVAL
import httplib from httplib import HTTPConnection
from portage.output import white, yellow, turquoise, green from portage.output import white, yellow, turquoise, green
from portage.exception import AmbiguousPackageName from portage.exception import AmbiguousPackageName
@ -37,6 +37,8 @@ from euscan.out import progress_bar
# Globals # Globals
isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty()
def exit_helper(status): def exit_helper(status):
if CONFIG["format"]: if CONFIG["format"]:
@ -52,7 +54,7 @@ def setup_signals():
signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN)
print() print()
exit_helper(errno.EINTR) exit_helper(EINTR)
signal.signal(signal.SIGINT, exithandler) signal.signal(signal.SIGINT, exithandler)
signal.signal(signal.SIGTERM, exithandler) signal.signal(signal.SIGTERM, exithandler)
@ -121,6 +123,8 @@ def print_usage(_error=None, help=None):
print(yellow(" -f, --format=<format>") + print(yellow(" -f, --format=<format>") +
" - define the output " + yellow("<format>") + " - define the output " + yellow("<format>") +
" (available: json)", file=out) " (available: json)", file=out)
print(yellow(" -p, --progress") +
" - display a progress bar", file=out)
print(file=out) print(file=out)
if _error in ('packages',) or help: if _error in ('packages',) or help:
@ -172,6 +176,8 @@ def parse_args():
CONFIG['format'] = a CONFIG['format'] = a
CONFIG['nocolor'] = True CONFIG['nocolor'] = True
pp.output.nocolor() pp.output.nocolor()
elif o in ("-p", "--progress"):
CONFIG['progress'] = isatty
else: else:
return_code = False return_code = False
@ -179,7 +185,7 @@ def parse_args():
# here are the different allowed command line options (getopt args) # here are the different allowed command line options (getopt args)
getopt_options = {'short': {}, 'long': {}} getopt_options = {'short': {}, 'long': {}}
getopt_options['short']['global'] = "hVCqv1bf:" getopt_options['short']['global'] = "hVCqv1bf:p"
getopt_options['long']['global'] = [ getopt_options['long']['global'] = [
"help", "version", "nocolor", "quiet", "verbose", "oneshot", "help", "version", "nocolor", "quiet", "verbose", "oneshot",
"brute-force=", "format=" "brute-force=", "format="
@ -207,7 +213,7 @@ def parse_args():
def main(): def main():
"""Parse command line and execute all actions.""" """Parse command line and execute all actions."""
CONFIG['nocolor'] = ( CONFIG['nocolor'] = (
port_settings["NOCOLOR"] in ('yes', 'true') or not sys.stdout.isatty() port_settings["NOCOLOR"] in ('yes', 'true') or not isatty
) )
if CONFIG['nocolor']: if CONFIG['nocolor']:
pp.output.nocolor() pp.output.nocolor()
@ -230,19 +236,23 @@ def main():
else: else:
print_usage(e.value) print_usage(e.value)
exit_helper(errno.EINVAL) exit_helper(EINVAL)
if CONFIG['verbose'] > 2: if CONFIG['verbose'] > 2:
httplib.HTTPConnection.debuglevel = 1 HTTPConnection.debuglevel = 1
isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() if not CONFIG["format"]:
CONFIG["progress"] = False
for query in queries:
on_progress = None on_progress = None
if (CONFIG['format'] or CONFIG['quiet']) and isatty: if CONFIG['progress']:
print("%s:" % query, file=sys.stderr)
on_progress_gen = progress_bar() on_progress_gen = progress_bar()
on_progress = on_progress_gen.next() on_progress = on_progress_gen.next()
on_progress(maxval=len(queries) * 100, increment=0)
for query in queries:
if CONFIG["progress"]:
on_progress(increment=10, label=query)
ret = [] ret = []
@ -271,27 +281,16 @@ def main():
output.eerror('%s: %s' % (query, str(err))) output.eerror('%s: %s' % (query, str(err)))
exit_helper(1) exit_helper(1)
if not CONFIG['quiet'] and not CONFIG['format']: if not ret and not CONFIG['quiet']:
print()
if (CONFIG['format'] or CONFIG['quiet']) and isatty:
on_progress_gen.next()
print("\n", file=sys.stderr)
if ret is not None:
if len(ret) > 0:
for cp, url, version, handler, confidence in ret:
output.result(cp, version, url, handler, confidence)
if (CONFIG['format'] or CONFIG['quiet']) and isatty:
print("\n", file=sys.stderr)
elif not CONFIG['quiet']:
output.ewarn( output.ewarn(
"Didn't find any new version, check package's homepage " + "Didn't find any new version, check package's homepage " +
"for more informations" "for more informations"
) )
if CONFIG['progress']:
on_progress_gen.next()
print("\n", file=sys.stderr)
output.set_query(None) output.set_query(None)

40
euscanwww/README.Celery Normal file
View File

@ -0,0 +1,40 @@
Requirements
============
Installing requirements with setup.py is the easiest way, just run::
python setup.py develop
or::
python setup.py install # to install euscan and requirements
If you prefer to use portage just install dev-python/django-celery-2.5.5
There's the need of having a broker for tasks. The default and reccommended
broker is RabbitMQ.
Install with::
emerge rabbitmq-server
Running Celery
==============
You'll need:
* celeryd (celery daemon for running tasks)::
python manage.py celeryd -E -l INFO
* celerycam (for monitoring celery and see the results in the django admin page)::
python manage.py celerycam
* celerybeat (for running periodic tasks)::
python manage.py celerybeat -l INFO
TODO
====
Provide a supervisord configuration

View File

@ -1,22 +1,54 @@
from djeuscan.models import Package, Version, VersionLog, EuscanResult, \ from djeuscan.models import Package, Version, VersionLog, EuscanResult, \
Log, WorldLog, CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer Log, WorldLog, CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer, \
RefreshPackageQuery, HerdAssociation, CategoryAssociation, \
MaintainerAssociation, PackageAssociation
from django.contrib import admin from django.contrib import admin
admin.site.register(Herd)
admin.site.register(Maintainer) class EuscanResultAdmin(admin.ModelAdmin):
search_fields = ('package__name', 'package__category')
list_filter = ('datetime', )
ordering = ["-datetime"]
class HerdAdmin(admin.ModelAdmin):
search_fields = ('herd', 'email')
ordering = ["herd"]
class MaintainerAdmin(admin.ModelAdmin):
search_fields = ('name', 'email')
ordering = ["name"]
class PackageAdmin(admin.ModelAdmin): class PackageAdmin(admin.ModelAdmin):
search_fields = ('category', 'name') search_fields = ('category', 'name')
list_filter = ('category', )
class VersionAdmin(admin.ModelAdmin):
search_fields = ('package__name', 'package__category')
list_filter = ('overlay', 'packaged', 'alive')
admin.site.register(Package, PackageAdmin) admin.site.register(Package, PackageAdmin)
admin.site.register(Version) admin.site.register(Herd, HerdAdmin)
admin.site.register(Maintainer, MaintainerAdmin)
admin.site.register(Version, VersionAdmin)
admin.site.register(VersionLog) admin.site.register(VersionLog)
admin.site.register(EuscanResult)
admin.site.register(EuscanResult, EuscanResultAdmin)
admin.site.register(Log) admin.site.register(Log)
admin.site.register(WorldLog) admin.site.register(WorldLog)
admin.site.register(CategoryLog) admin.site.register(CategoryLog)
admin.site.register(HerdLog) admin.site.register(HerdLog)
admin.site.register(MaintainerLog) admin.site.register(MaintainerLog)
admin.site.register(RefreshPackageQuery)
admin.site.register(HerdAssociation)
admin.site.register(CategoryAssociation)
admin.site.register(MaintainerAssociation)
admin.site.register(PackageAssociation)

View File

@ -1,25 +1,10 @@
import logging
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from djeuscan import charts
from djeuscan.processing import set_verbosity_level
from djeuscan.processing.regen_rrds import regen_rrds
def regen_rrds(): logger = logging.getLogger(__name__)
"""
Regenerates the rrd database
"""
for wlog in WorldLog.objects.all():
charts.rrd_update('world', wlog.datetime, wlog)
for clog in CategoryLog.objects.all():
charts.rrd_update('category-%s' % clog.category,
clog.datetime, clog)
for hlog in HerdLog.objects.all():
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
for mlog in MaintainerLog.objects.all():
charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
mlog.datetime, mlog)
class Command(BaseCommand): class Command(BaseCommand):
@ -27,4 +12,5 @@ class Command(BaseCommand):
help = 'Regenerate rrd database' help = 'Regenerate rrd database'
def handle(self, *args, **options): def handle(self, *args, **options):
regen_rrds() set_verbosity_level(logger, options.get("verbosity", 1))
regen_rrds(logger=logger)

View File

@ -1,142 +1,13 @@
import sys import sys
import logging
from optparse import make_option from optparse import make_option
from gentoolkit.query import Query
from gentoolkit.errors import GentoolkitFatalError
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from django.core.management.color import color_style
from django.core.exceptions import ValidationError
from djeuscan.models import Package, Herd, Maintainer from djeuscan.processing import set_verbosity_level
from djeuscan.processing.scan_metadata import scan_metadata
logger = logging.getLogger(__name__)
class ScanMetadata(object):
def __init__(self, quiet):
self.quiet = quiet
self.style = color_style()
@commit_on_success
def run(self, query=None, obj=None):
matches = Query(query).find(
include_masked=True,
in_installed=False,
)
if not matches:
sys.stderr.write(
self.style.ERROR("Unknown package '%s'\n" % query)
)
return
matches = sorted(matches)
pkg = matches.pop()
if '9999' in pkg.version and len(matches):
pkg = matches.pop()
if not obj:
obj, created = Package.objects.get_or_create(
category=pkg.category, name=pkg.name
)
else:
created = False
try:
obj.homepage = pkg.environment("HOMEPAGE")
obj.description = pkg.environment("DESCRIPTION")
except GentoolkitFatalError, err:
sys.stderr.write(
self.style.ERROR(
"Gentoolkit fatal error: '%s'\n" % str(err)
)
)
if created and not self.quiet:
sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
if pkg.metadata:
herds = dict(
[(herd[0], herd) for herd in pkg.metadata.herds(True)]
)
maintainers = dict(
[(m.email, m) for m in pkg.metadata.maintainers()]
)
existing_herds = [h.herd for h in obj.herds.all()]
new_herds = set(herds.keys()).difference(existing_herds)
old_herds = set(existing_herds).difference(herds.keys())
existing_maintainers = [m.email for m in obj.maintainers.all()]
new_maintainers = set(
maintainers.keys()).difference(existing_maintainers
)
old_maintainers = set(
existing_maintainers).difference(maintainers.keys()
)
for herd in obj.herds.all():
if herd.herd in old_herds:
obj.herds.remove(herd)
for herd in new_herds:
herd = self.store_herd(*herds[herd])
obj.herds.add(herd)
for maintainer in obj.maintainers.all():
if maintainer.email in old_maintainers:
obj.maintainers.remove(maintainer)
for maintainer in new_maintainers:
maintainer = maintainers[maintainer]
try:
maintainer = self.store_maintainer(
maintainer.name, maintainer.email
)
obj.maintainers.add(maintainer)
except ValidationError:
sys.stderr.write(
self.style.ERROR("Bad maintainer: '%s' '%s'\n" % (maintainer.name, maintainer.email))
)
obj.save()
def store_herd(self, name, email):
if not name:
name = '{nil}'
name = name.strip("\r").strip("\n").strip("\t").strip()
herd, created = Herd.objects.get_or_create(
herd=name,
defaults={"email": email}
)
if created and not self.quiet:
sys.stdout.write('+ [h] %s <%s>\n' % (name, email))
herd.email = email
herd.save()
return herd
def store_maintainer(self, name, email):
if not name:
name = email
if not name:
name = '{nil}'
maintainer, created = Maintainer.objects.get_or_create(
email=email,
defaults={"name": name}
)
if created:
if not self.quiet:
sys.stdout.write(
'+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
)
return maintainer
class Command(BaseCommand): class Command(BaseCommand):
@ -148,26 +19,19 @@ class Command(BaseCommand):
dest='all', dest='all',
default=False, default=False,
help='Scan all packages'), help='Scan all packages'),
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
) )
args = '<package package ...>' args = '<package package ...>'
help = 'Scans metadata and fills database' help = 'Scans metadata and fills database'
def handle(self, *args, **options): def handle(self, *args, **options):
self.options = options set_verbosity_level(logger, options.get("verbosity", 1))
scan_metadata = ScanMetadata(quiet=options["quiet"])
if options['all']: if options['all']:
for pkg in Package.objects.all(): packages = None
scan_metadata.run('%s/%s' % (pkg.category, pkg.name), pkg)
elif len(args) > 0: elif len(args):
for package in args: packages = [pkg for pkg in args]
scan_metadata.run(package)
else: else:
for package in sys.stdin.readlines(): packages = [pkg[:-1] for pkg in sys.stdin.readlines()]
scan_metadata.run(package[:-1])
scan_metadata(packages=packages, logger=logger)

View File

@ -1,274 +1,13 @@
import subprocess
import portage
import sys import sys
import os import logging
import re
from optparse import make_option from optparse import make_option
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from django.core.management.color import color_style
from djeuscan.models import Package, Version, VersionLog from djeuscan.processing import set_verbosity_level
from djeuscan.processing.scan_portage import scan_portage
logger = logging.getLogger(__name__)
class ScanPortage(object):
def __init__(self, stdout=None, **options):
if stdout is None:
self.stdout = sys.stdout
else:
self.stdout = stdout
self.options = options
self.style = color_style()
self._cache = {'packages': {}, 'versions': {}}
self._overlays = None
def cache_hash_package(self, category, name):
return '%s/%s' % (category, name)
def cache_store_package(self, package):
key = self.cache_hash_package(package.category, package.name)
self._cache['packages'][key] = package
def cache_get_package(self, category, name):
return self._cache['packages'].get(
self.cache_hash_package(category, name)
)
def cache_hash_version(self, category, name, version, revision, slot,
overlay):
key = '%s/%s-%s-r%s %s %s' % (category, name,
version, revision,
slot, overlay)
return key
def cache_get_version(self, category, name, version, revision, slot,
overlay):
key = self.cache_hash_version(category, name, version, revision, slot,
overlay)
return self._cache['versions'].get(key)
def cache_store_version(self, version):
key = self.cache_hash_version(
version.package.category, version.package.name, version.version,
version.revision, version.slot, version.overlay
)
self._cache['versions'][key] = version
def overlays(self):
if self._overlays:
return self._overlays
env = os.environ
env['OVERLAYS_LIST'] = 'all'
env['PRINT_COUNT_ALWAYS'] = 'never'
cmd = ['eix', '-!']
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
output = output.strip().strip('\n').split('\n')
overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
self._overlays = {}
for line in output:
match = overlay_re.match(line)
if not match:
continue
self._overlays[match.group('key')] = match.group('name')
return self._overlays
@commit_on_success
def run(self, query=None):
env = os.environ
env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages',
'-x']
if query:
cmd.extend(['--exact', query])
if self.options['all']:
if not self.options['quiet']:
self.stdout.write('Killing existing versions...')
self.stdout.flush()
Version.objects.filter(packaged=True).update(alive=False)
if not self.options['quiet']:
self.stdout.write('done\n')
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
output = output.strip().strip('\n')
if len(output) == 0:
if not query:
return
if self.options['purge-packages']:
if not self.options['quiet']:
sys.stdout.write('- [p] %s\n' % (query))
if '/' in query:
cat, pkg = portage.catsplit(query)
Package.objects.filter(category=cat, name=pkg).delete()
else:
Package.objects.filter(name=query).delete()
else:
sys.stderr.write(
self.style.ERROR(
"Unknown package '%s'\n" % query
)
)
return
output = output.split('\n')
packages = {}
line_re = re.compile(
r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$'
)
package = None
for line in output:
match = line_re.match(line)
if not match:
continue
cpv = match.group('cpv')
slot = match.group('slot')
overlay = match.group('overlay')
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
packages['%s/%s' % (cat, pkg)] = True
if not package or not \
(cat == package.category and pkg == package.name):
package = self.store_package(cat, pkg)
self.store_version(package, cpv, slot, overlay)
if self.options['purge-packages'] and not query:
for package in Package.objects.all():
cp = "%s/%s" % (package.category, package.name)
if cp not in packages:
if not self.options['quiet']:
sys.stdout.write('- [p] %s\n' % (package))
package.delete()
def store_package(self, cat, pkg):
created = False
obj = self.cache_get_package(cat, pkg)
if not obj:
obj, created = Package.objects.get_or_create(
category=cat,
name=pkg
)
self.cache_store_package(obj)
if created:
if not self.options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
# Set all versions dead, then set found versions alive and
# delete old versions
if not self.options['all']:
Version.objects.filter(
package=obj,
packaged=True
).update(alive=False)
return obj
def store_version(self, package, cpv, slot, overlay):
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
overlays = self.overlays()
if overlay in overlays:
overlay = overlays[overlay]
else:
overlay = 'gentoo'
created = False
obj = self.cache_get_version(
package.category, package.name, ver, rev, slot, overlay
)
if not obj:
obj, created = Version.objects.get_or_create(
package=package, slot=slot,
revision=rev, version=ver,
overlay=overlay,
defaults={"alive": True, "packaged": True}
)
if not created: # Created objects have defaults values
obj.alive = True
obj.packaged = True
obj.save()
if created:
self.cache_store_version(obj)
# nothing to do (note: it can't be an upstream version because
# overlay can't be empty here)
if not created:
return
if not self.options['quiet']:
sys.stdout.write('+ [v] %s \n' % (obj))
if overlay == 'gentoo':
package.n_packaged += 1
else:
package.n_overlay += 1
package.n_versions += 1
package.save()
if self.options['no-log']:
return
VersionLog.objects.create(
package=obj.package,
action=VersionLog.VERSION_ADDED,
slot=obj.slot,
revision=obj.revision,
version=obj.version,
overlay=obj.overlay
)
@commit_on_success
def purge_versions(options):
# For each dead versions
for version in Version.objects.filter(packaged=True, alive=False):
if version.overlay == 'gentoo':
version.package.n_packaged -= 1
else:
version.package.n_overlay -= 1
version.package.n_versions -= 1
version.package.save()
if not options['quiet']:
sys.stdout.write('- [v] %s\n' % (version))
if options['no-log']:
continue
VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED,
slot=version.slot,
revision=version.revision,
version=version.version,
overlay=version.overlay
)
Version.objects.filter(packaged=True, alive=False).delete()
class Command(BaseCommand): class Command(BaseCommand):
@ -301,43 +40,26 @@ class Command(BaseCommand):
default=False, default=False,
help=('Prefetch all versions and packages from DB to ' help=('Prefetch all versions and packages from DB to '
'speedup full scan process.')), 'speedup full scan process.')),
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
) )
args = '[package package ...]' args = '[package package ...]'
help = 'Scans portage tree and fills database' help = 'Scans portage tree and fills database'
def handle(self, *args, **options): def handle(self, *args, **options):
scan_portage = ScanPortage(stdout=self.stdout, **options) set_verbosity_level(logger, options.get("verbosity", 1))
if not options['quiet']:
self.stdout.write('Scanning portage tree...\n')
if options['prefetch']:
if not options['quiet']:
self.stdout.write('Prefetching objects...')
self.stdout.flush()
for package in Package.objects.all():
scan_portage.cache_store_package(package)
for version in Version.objects.select_related('package').all():
scan_portage.cache_store_version(version)
if not options['quiet']:
self.stdout.write('done\n')
if options['all']: if options['all']:
scan_portage.run() packages = None
elif len(args): elif len(args):
for package in args: packages = [pkg for pkg in args]
scan_portage.run(package)
else: else:
for package in sys.stdin.readlines(): packages = [pkg[:-1] for pkg in sys.stdin.readlines()]
scan_portage.run(package[:-1])
if options['purge-versions']: scan_portage(
purge_versions(options) packages=packages,
no_log=options["no-log"],
if not options['quiet']: purge_packages=options["purge-packages"],
self.stdout.write('Done.\n') purge_versions=options["purge-versions"],
prefetch=options["prefetch"],
logger=logger,
)

View File

@ -1,151 +1,13 @@
import subprocess import logging
import portage
import sys import sys
import re
from StringIO import StringIO
from optparse import make_option from optparse import make_option
from collections import defaultdict
from django.utils import timezone
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from djeuscan.models import Package, Version, EuscanResult, VersionLog from djeuscan.processing import set_verbosity_level
from djeuscan.processing.scan_upstream import scan_upstream
logger = logging.getLogger(__name__)
class ScanUpstream(object):
def __init__(self, options=None):
if options is None:
self.options = defaultdict(None)
else:
self.options = options
def run(self, packages=None):
for package in packages:
cmd = ['euscan', package]
fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = StringIO(fp.communicate()[0])
self.parse_output(output)
def store_result(self, package, log):
# Remove previous logs
EuscanResult.objects.filter(package=package).delete()
obj = EuscanResult()
obj.package = package
obj.result = log
obj.datetime = timezone.now()
obj.save()
def store_package(self, cpv):
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
if created and not self.options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
# Set all versions dead, then set found versions alive and
# delete old versions
Version.objects.filter(package=obj, packaged=False).update(alive=False)
return obj
def store_version(self, package, ver, url):
obj, created = Version.objects.get_or_create(
package=package, slot='', revision='r0', version=ver, overlay='',
defaults={"alive": True, "urls": url, "packaged": False}
)
if not created:
obj.alive = True
obj.urls = url
obj.packaged = False
obj.save()
# If it's not a new version, just update the object and continue
if not created:
return
if not self.options['quiet']:
sys.stdout.write('+ [u] %s %s\n' % (obj, url))
VersionLog.objects.create(
package=package,
action=VersionLog.VERSION_ADDED,
slot='',
revision='r0',
version=ver,
overlay=''
)
package.n_versions += 1
package.save()
@commit_on_success
def parse_output(self, output):
from portage.versions import _cp
if type(_cp) == dict:
_cp = _cp["dots_allowed_in_PN"]
package_re = re.compile(
r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$'
)
version_re = re.compile(
r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$'
)
package = None
log = ""
while True:
line = output.readline()
if line == '':
break
match = package_re.match(line)
if match:
if package:
self.store_result(package, log)
cpv = match.group('cpv')
package = self.store_package(cpv)
log = line
continue
log += line
match = version_re.match(line)
if match:
ver = match.group('ver')
url = match.group('url')
self.store_version(package, ver, url)
if package:
self.store_result(package, log)
@commit_on_success
def purge_versions(options):
# For each dead versions
for version in Version.objects.filter(packaged=False, alive=False):
VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED,
slot=version.slot,
revision=version.revision,
version=version.version,
overlay=version.overlay
)
version.package.n_versions -= 1
version.package.save()
if not options['quiet']:
sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
Version.objects.filter(packaged=False, alive=False).delete()
class Command(BaseCommand): class Command(BaseCommand):
@ -162,41 +24,23 @@ class Command(BaseCommand):
dest='purge-versions', dest='purge-versions',
default=False, default=False,
help='Purge old versions'), help='Purge old versions'),
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
) )
args = '<package package ...>' args = '<package package ...>'
help = 'Scans metadata and fills database' help = 'Scans metadata and fills database'
def handle(self, *args, **options): def handle(self, *args, **options):
scan_upstream = ScanUpstream(options) set_verbosity_level(logger, options.get("verbosity", 1))
if not args and not options['all']:
scan_upstream.parse_output(sys.stdin)
if options['purge-versions']:
purge_versions(options)
return
if not options['quiet']:
self.stdout.write('Scanning upstream...\n')
packages = []
if options['all']: if options['all']:
for pkg in Package.objects.all(): packages = None
packages.append('%s/%s' % (pkg.category, pkg.name))
elif args: elif len(args):
packages = list(args) packages = [pkg for pkg in args]
else: else:
packages = [package[:-1] for package in sys.stdin.readlines()] packages = [pkg[:-1] for pkg in sys.stdin.readlines()]
scan_upstream.run(packages) scan_upstream(
packages=packages,
if options['purge-versions']: purge_versions=options["purge-versions"],
purge_versions(options) logger=logger,
)
if not options['quiet']:
self.stdout.write('Done.\n')

View File

@ -1,185 +1,12 @@
import sys import logging
from optparse import make_option from optparse import make_option
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from django.utils import timezone
from djeuscan.models import Package, Herd, Maintainer, Version from djeuscan.processing import set_verbosity_level
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog from djeuscan.processing.update_counters import update_counters
from djeuscan import charts
from distutils.version import StrictVersion, LooseVersion logger = logging.getLogger(__name__)
def compare_versions(version1, version2):
try:
return cmp(StrictVersion(version1), StrictVersion(version2))
# in case of abnormal version number, fall back to LooseVersion
except ValueError:
return cmp(LooseVersion(version1), LooseVersion(version2))
def add_safe(storage, key):
if key not in storage:
storage[key] = 1
else:
storage[key] += 1
def add_last_ver(storage, version):
key = version['package_id']
if key not in storage:
storage[key] = version
return
if version['version'].startswith('9999'):
return
if compare_versions(storage[key]['version'],
version['version']) < 0:
storage[key] = version
@commit_on_success
def update_counters(stdout=None, **options):
if stdout is None:
stdout = sys.stdout
now = timezone.now()
categories = {}
herds = {}
maintainers = {}
wlog = None
if not options['nolog']:
wlog = WorldLog()
wlog.datetime = now
for cat in Package.objects.values('category').distinct():
clog = CategoryLog()
clog.datetime = now
clog.category = cat['category']
categories[clog.category] = clog
for herd in Herd.objects.all():
hlog = HerdLog()
hlog.datetime = now
hlog.herd = herd
herds[herd.id] = hlog
for maintainer in Maintainer.objects.all():
mlog = MaintainerLog()
mlog.datetime = now
mlog.maintainer = maintainer
maintainers[maintainer.id] = mlog
package_queryset = Package.objects.all()
n_versions = {}
n_packaged = {}
n_overlay = {}
last_versions_gentoo = {}
last_versions_overlay = {}
last_versions_upstream = {}
if not options['fast']:
attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
for version in Version.objects.all().values(*attrs):
overlay, packaged = version['overlay'], version['packaged']
package_id = version['package_id']
add_safe(n_versions, package_id)
if not packaged:
add_last_ver(last_versions_upstream, version)
continue
if overlay == 'gentoo':
add_safe(n_packaged, package_id)
add_last_ver(last_versions_gentoo, version)
else:
add_safe(n_overlay, package_id)
add_last_ver(last_versions_overlay, version)
for package in package_queryset.select_related('herds', 'maintainers'):
if not options['fast']:
package.n_versions = n_versions.get(package.id, 0)
package.n_packaged = n_packaged.get(package.id, 0)
package.n_overlay = n_overlay.get(package.id, 0)
default = {'id': None}
package.last_version_gentoo_id = last_versions_gentoo.get(
package.id, default
)['id']
package.last_version_overlay_id = last_versions_overlay.get(
package.id, default
)['id']
package.last_version_upstream_id = last_versions_upstream.get(
package.id, default
)['id']
package.save()
n_packages_gentoo = int(package.n_packaged == package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged \
+ package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay \
< package.n_versions)
def update_row(storage, key):
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_outdated += n_packages_outdated
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
def update_log(storage, qs):
for row in qs:
update_row(storage, row['id'])
if not options['nolog']:
update_log(herds, package.herds.all().values('id'))
update_log(maintainers, package.maintainers.all().values('id'))
update_row(categories, package.category)
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
if options['nolog']:
return
for clog in categories.values():
if not options['quiet']:
stdout.write('+ [cl] %s\n' % clog)
charts.rrd_update('category-%s' % clog.category, now, clog)
clog.save()
for hlog in herds.values():
if not options['quiet']:
stdout.write('+ [hl] %s\n' % hlog)
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
hlog.save()
for mlog in maintainers.values():
if not options['quiet']:
stdout.write('+ [ml] %s\n' % mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save()
charts.rrd_update('world', now, wlog)
wlog.save()
class Command(BaseCommand): class Command(BaseCommand):
@ -187,11 +14,6 @@ class Command(BaseCommand):
help = 'Update counters' help = 'Update counters'
option_list = BaseCommand.option_list + ( option_list = BaseCommand.option_list + (
make_option('--quiet',
action='store_true',
dest='quiet',
default=False,
help='Be quiet'),
make_option('--fast', make_option('--fast',
action='store_true', action='store_true',
dest='fast', dest='fast',
@ -205,4 +27,9 @@ class Command(BaseCommand):
) )
def handle(self, *args, **options): def handle(self, *args, **options):
update_counters(stdout=self.stdout, **options) set_verbosity_level(logger, options.get("verbosity", 1))
update_counters(
fast=options["fast"],
nolog=options["nolog"],
logger=logger,
)

View File

@ -35,7 +35,6 @@ ANNOTATE_DICT = {name: models.Sum(name)
class PackageMixin(object): class PackageMixin(object):
for_maintainer = _gen_for_function('maintainers') for_maintainer = _gen_for_function('maintainers')
for_herd = _gen_for_function('herds') for_herd = _gen_for_function('herds')
for_category = _gen_for_function('category') for_category = _gen_for_function('category')

View File

@ -0,0 +1,139 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'RefreshPackageQuery'
db.create_table('djeuscan_refreshpackagequery', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('query', self.gf('django.db.models.fields.CharField')(unique=True, max_length=256)),
('priority', self.gf('django.db.models.fields.IntegerField')(default=0)),
))
db.send_create_signal('djeuscan', ['RefreshPackageQuery'])
# Changing field 'Package.last_version_gentoo'
db.alter_column('djeuscan_package', 'last_version_gentoo_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['djeuscan.Version']))
# Changing field 'Package.last_version_overlay'
db.alter_column('djeuscan_package', 'last_version_overlay_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['djeuscan.Version']))
# Changing field 'Package.last_version_upstream'
db.alter_column('djeuscan_package', 'last_version_upstream_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['djeuscan.Version']))
def backwards(self, orm):
# Deleting model 'RefreshPackageQuery'
db.delete_table('djeuscan_refreshpackagequery')
# Changing field 'Package.last_version_gentoo'
db.alter_column('djeuscan_package', 'last_version_gentoo_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['djeuscan.Version']))
# Changing field 'Package.last_version_overlay'
db.alter_column('djeuscan_package', 'last_version_overlay_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['djeuscan.Version']))
# Changing field 'Package.last_version_upstream'
db.alter_column('djeuscan_package', 'last_version_upstream_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['djeuscan.Version']))
models = {
'djeuscan.categorylog': {
'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.euscanresult': {
'Meta': {'object_name': 'EuscanResult'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'djeuscan.herd': {
'Meta': {'object_name': 'Herd'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'djeuscan.herdlog': {
'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.log': {
'Meta': {'object_name': 'Log'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'djeuscan.maintainer': {
'Meta': {'object_name': 'Maintainer'},
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.maintainerlog': {
'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
},
'djeuscan.package': {
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.refreshpackagequery': {
'Meta': {'object_name': 'RefreshPackageQuery'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
},
'djeuscan.version': {
'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.versionlog': {
'Meta': {'object_name': 'VersionLog'},
'action': ('django.db.models.fields.IntegerField', [], {}),
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.worldlog': {
'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
}
}
complete_apps = ['djeuscan']

View File

@ -0,0 +1,238 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'CategoryAssociation'
db.create_table('djeuscan_categoryassociation', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
('category', self.gf('django.db.models.fields.CharField')(max_length=128)),
))
db.send_create_signal('djeuscan', ['CategoryAssociation'])
# Adding unique constraint on 'CategoryAssociation', fields ['user', 'category']
db.create_unique('djeuscan_categoryassociation', ['user_id', 'category'])
# Adding model 'PackageAssociation'
db.create_table('djeuscan_packageassociation', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
('package', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Package'])),
))
db.send_create_signal('djeuscan', ['PackageAssociation'])
# Adding unique constraint on 'PackageAssociation', fields ['user', 'package']
db.create_unique('djeuscan_packageassociation', ['user_id', 'package_id'])
# Adding model 'HerdAssociation'
db.create_table('djeuscan_herdassociation', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
('herd', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Herd'])),
))
db.send_create_signal('djeuscan', ['HerdAssociation'])
# Adding unique constraint on 'HerdAssociation', fields ['user', 'herd']
db.create_unique('djeuscan_herdassociation', ['user_id', 'herd_id'])
# Adding model 'MaintainerAssociation'
db.create_table('djeuscan_maintainerassociation', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
('maintainer', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Maintainer'])),
))
db.send_create_signal('djeuscan', ['MaintainerAssociation'])
# Adding unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer']
db.create_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id'])
def backwards(self, orm):
# Removing unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer']
db.delete_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id'])
# Removing unique constraint on 'HerdAssociation', fields ['user', 'herd']
db.delete_unique('djeuscan_herdassociation', ['user_id', 'herd_id'])
# Removing unique constraint on 'PackageAssociation', fields ['user', 'package']
db.delete_unique('djeuscan_packageassociation', ['user_id', 'package_id'])
# Removing unique constraint on 'CategoryAssociation', fields ['user', 'category']
db.delete_unique('djeuscan_categoryassociation', ['user_id', 'category'])
# Deleting model 'CategoryAssociation'
db.delete_table('djeuscan_categoryassociation')
# Deleting model 'PackageAssociation'
db.delete_table('djeuscan_packageassociation')
# Deleting model 'HerdAssociation'
db.delete_table('djeuscan_herdassociation')
# Deleting model 'MaintainerAssociation'
db.delete_table('djeuscan_maintainerassociation')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'djeuscan.categoryassociation': {
'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.categorylog': {
'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.euscanresult': {
'Meta': {'object_name': 'EuscanResult'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'djeuscan.herd': {
'Meta': {'object_name': 'Herd'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'djeuscan.herdassociation': {
'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.herdlog': {
'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.log': {
'Meta': {'object_name': 'Log'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'djeuscan.maintainer': {
'Meta': {'object_name': 'Maintainer'},
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.maintainerassociation': {
'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.maintainerlog': {
'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
},
'djeuscan.package': {
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.packageassociation': {
'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.refreshpackagequery': {
'Meta': {'object_name': 'RefreshPackageQuery'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
},
'djeuscan.version': {
'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.versionlog': {
'Meta': {'object_name': 'VersionLog'},
'action': ('django.db.models.fields.IntegerField', [], {}),
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.worldlog': {
'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
}
}
complete_apps = ['djeuscan']

View File

@ -0,0 +1,197 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'Version.version_type'
db.add_column('djeuscan_version', 'version_type',
self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True),
keep_default=False)
# Adding field 'Version.handler'
db.add_column('djeuscan_version', 'handler',
self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True),
keep_default=False)
# Adding field 'Version.confidence'
db.add_column('djeuscan_version', 'confidence',
self.gf('django.db.models.fields.IntegerField')(default=0),
keep_default=False)
def backwards(self, orm):
# Deleting field 'Version.version_type'
db.delete_column('djeuscan_version', 'version_type')
# Deleting field 'Version.handler'
db.delete_column('djeuscan_version', 'handler')
# Deleting field 'Version.confidence'
db.delete_column('djeuscan_version', 'confidence')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'djeuscan.categoryassociation': {
'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.categorylog': {
'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.euscanresult': {
'Meta': {'object_name': 'EuscanResult'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'djeuscan.herd': {
'Meta': {'object_name': 'Herd'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'djeuscan.herdassociation': {
'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.herdlog': {
'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.log': {
'Meta': {'object_name': 'Log'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'djeuscan.maintainer': {
'Meta': {'object_name': 'Maintainer'},
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.maintainerassociation': {
'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.maintainerlog': {
'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
},
'djeuscan.package': {
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.packageassociation': {
'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.refreshpackagequery': {
'Meta': {'object_name': 'RefreshPackageQuery'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
},
'djeuscan.version': {
'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'version_type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
},
'djeuscan.versionlog': {
'Meta': {'object_name': 'VersionLog'},
'action': ('django.db.models.fields.IntegerField', [], {}),
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.worldlog': {
'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
}
}
complete_apps = ['djeuscan']

View File

@ -0,0 +1,191 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'EuscanResult.scan_time'
db.add_column('djeuscan_euscanresult', 'scan_time',
self.gf('django.db.models.fields.FloatField')(null=True, blank=True),
keep_default=False)
# Adding field 'EuscanResult.ebuild'
db.add_column('djeuscan_euscanresult', 'ebuild',
self.gf('django.db.models.fields.CharField')(default='', max_length=256, blank=True),
keep_default=False)
def backwards(self, orm):
# Deleting field 'EuscanResult.scan_time'
db.delete_column('djeuscan_euscanresult', 'scan_time')
# Deleting field 'EuscanResult.ebuild'
db.delete_column('djeuscan_euscanresult', 'ebuild')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'djeuscan.categoryassociation': {
'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.categorylog': {
'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.euscanresult': {
'Meta': {'object_name': 'EuscanResult'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'})
},
'djeuscan.herd': {
'Meta': {'object_name': 'Herd'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'djeuscan.herdassociation': {
'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.herdlog': {
'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.log': {
'Meta': {'object_name': 'Log'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'djeuscan.maintainer': {
'Meta': {'object_name': 'Maintainer'},
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.maintainerassociation': {
'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.maintainerlog': {
'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
},
'djeuscan.package': {
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.packageassociation': {
'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.refreshpackagequery': {
'Meta': {'object_name': 'RefreshPackageQuery'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
},
'djeuscan.version': {
'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'version_type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
},
'djeuscan.versionlog': {
'Meta': {'object_name': 'VersionLog'},
'action': ('django.db.models.fields.IntegerField', [], {}),
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.worldlog': {
'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
}
}
complete_apps = ['djeuscan']

View File

@ -2,6 +2,8 @@ from django.db import models
from django.core.validators import RegexValidator, validate_email, URLValidator from django.core.validators import RegexValidator, validate_email, URLValidator
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from django.contrib.auth.models import User
from djeuscan.managers import PackageManager, VersionLogManager, \ from djeuscan.managers import PackageManager, VersionLogManager, \
EuscanResultManager EuscanResultManager
@ -115,16 +117,19 @@ class Version(models.Model):
""" """
package = models.ForeignKey(Package) package = models.ForeignKey(Package)
slot = models.CharField(max_length=128, blank=True, default='') slot = models.CharField(max_length=128, blank=True, default="")
revision = models.CharField(max_length=128) revision = models.CharField(max_length=128)
version = models.CharField(max_length=128) version = models.CharField(max_length=128)
packaged = models.BooleanField() packaged = models.BooleanField()
overlay = models.CharField(max_length=128, blank=True, overlay = models.CharField(max_length=128, default='gentoo', db_index=True,
default='gentoo', db_index=True, validators=[validate_name], blank=True)
validators=[validate_name])
urls = models.TextField(blank=True) urls = models.TextField(blank=True)
alive = models.BooleanField(default=True, db_index=True) alive = models.BooleanField(default=True, db_index=True)
version_type = models.CharField(max_length=128, blank=True)
handler = models.CharField(max_length=128, blank=True)
confidence = models.IntegerField(default=0)
class Meta: class Meta:
unique_together = ['package', 'slot', 'revision', 'version', 'overlay'] unique_together = ['package', 'slot', 'revision', 'version', 'overlay']
@ -149,12 +154,12 @@ class VersionLog(models.Model):
package = models.ForeignKey(Package) package = models.ForeignKey(Package)
datetime = models.DateTimeField(auto_now_add=True) datetime = models.DateTimeField(auto_now_add=True)
slot = models.CharField(max_length=128, blank=True, default='') slot = models.CharField(max_length=128, blank=True, default="")
revision = models.CharField(max_length=128) revision = models.CharField(max_length=128)
version = models.CharField(max_length=128) version = models.CharField(max_length=128)
packaged = models.BooleanField() packaged = models.BooleanField()
overlay = models.CharField(max_length=128, blank=True, overlay = models.CharField(max_length=128, default='gentoo',
default='gentoo', validators=[validate_name]) validators=[validate_name], blank=True)
action = models.IntegerField(choices=VERSION_ACTIONS) action = models.IntegerField(choices=VERSION_ACTIONS)
objects = VersionLogManager() objects = VersionLogManager()
@ -182,6 +187,9 @@ class EuscanResult(models.Model):
datetime = models.DateTimeField() datetime = models.DateTimeField()
result = models.TextField(blank=True) result = models.TextField(blank=True)
scan_time = models.FloatField(null=True, blank=True)
ebuild = models.CharField(blank=True, max_length=256)
objects = EuscanResultManager() objects = EuscanResultManager()
class Meta: class Meta:
@ -191,6 +199,11 @@ class EuscanResult(models.Model):
self.full_clean() self.full_clean()
super(EuscanResult, self).save(*args, **kwargs) super(EuscanResult, self).save(*args, **kwargs)
def __unicode__(self):
return '[%s] %s/%s' % (
self.datetime, self.package.category, self.package.name
)
class Log(models.Model): class Log(models.Model):
""" """
@ -253,3 +266,55 @@ class MaintainerLog(Log):
def __unicode__(self): def __unicode__(self):
return u'%s %s' % (self.maintainer, Log.__unicode__(self)) return u'%s %s' % (self.maintainer, Log.__unicode__(self))
class RefreshPackageQuery(models.Model):
query = models.CharField(max_length=256, unique=True)
priority = models.IntegerField(default=0)
def __unicode__(self):
return u'[%d] %s' % (self.priority, self.query)
class HerdAssociation(models.Model):
user = models.ForeignKey(User)
herd = models.ForeignKey(Herd)
class Meta:
unique_together = ['user', 'herd']
def __unicode__(self):
return u'[%s] %s' % (self.user, self.herd)
class MaintainerAssociation(models.Model):
user = models.ForeignKey(User)
maintainer = models.ForeignKey(Maintainer)
class Meta:
unique_together = ['user', 'maintainer']
def __unicode__(self):
return u'[%s] %s' % (self.user, self.maintainer)
class PackageAssociation(models.Model):
user = models.ForeignKey(User)
package = models.ForeignKey(Package)
class Meta:
unique_together = ['user', 'package']
def __unicode__(self):
return u'[%s] %s' % (self.user, self.package)
class CategoryAssociation(models.Model):
user = models.ForeignKey(User)
category = models.CharField(max_length=128, validators=[validate_category])
class Meta:
unique_together = ['user', 'category']
def __unicode__(self):
return u'[%s] %s' % (self.user, self.category)

View File

@ -0,0 +1,31 @@
import logging
class FakeLogger(object):
def __getattr__(self, key):
return lambda *x, **y: None
def set_verbosity_level(logger, verbosity):
try:
verbosity = int(verbosity)
except (ValueError, TypeError):
return logger
levels = {
0: logging.DEBUG,
1: logging.INFO,
2: logging.WARNING,
3: logging.ERROR,
4: logging.CRITICAL
}
if verbosity < 0:
verbosity = 0
if verbosity > 4:
verbosity = 4
logger.setLevel(levels[verbosity])
return logger

View File

@ -0,0 +1,31 @@
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from djeuscan import charts
from djeuscan.processing import FakeLogger
def regen_rrds(logger=None):
"""
Regenerates the rrd database
"""
if logger is None:
logger = FakeLogger()
logger.info("Regenering RRDs for world")
for wlog in WorldLog.objects.all():
charts.rrd_update('world', wlog.datetime, wlog)
logger.info("Regenering RRDs for categories")
for clog in CategoryLog.objects.all():
charts.rrd_update('category-%s' % clog.category,
clog.datetime, clog)
logger.info("Regenering RRDs for herds")
for hlog in HerdLog.objects.all():
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
logger.info("Regenering RRDs for maintainers")
for mlog in MaintainerLog.objects.all():
charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
mlog.datetime, mlog)

View File

@ -0,0 +1,148 @@
from gentoolkit.query import Query
from gentoolkit.errors import GentoolkitFatalError
from django.db.transaction import commit_on_success
from django.core.management.color import color_style
from django.core.exceptions import ValidationError
from djeuscan.models import Package, Herd, Maintainer
from djeuscan.processing import FakeLogger
class ScanMetadata(object):
def __init__(self, logger=None):
self.style = color_style()
self.logger = logger or FakeLogger()
@commit_on_success
def scan(self, query=None, obj=None):
matches = Query(query).find(
include_masked=True,
in_installed=False,
)
if not matches:
self.logger.error(
self.style.ERROR("Unknown package '%s'" % query)
)
return
matches = sorted(matches)
pkg = matches.pop()
if '9999' in pkg.version and len(matches):
pkg = matches.pop()
if not obj:
obj, created = Package.objects.get_or_create(
category=pkg.category, name=pkg.name
)
else:
created = False
try:
obj.homepage = pkg.environment("HOMEPAGE")
obj.description = pkg.environment("DESCRIPTION")
except GentoolkitFatalError, err:
self.logger.error(
self.style.ERROR(
"Gentoolkit fatal error: '%s'" % str(err)
)
)
if created:
self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name))
if pkg.metadata:
herds = dict(
[(herd[0], herd) for herd in pkg.metadata.herds(True)]
)
maintainers = dict(
[(m.email, m) for m in pkg.metadata.maintainers()]
)
existing_herds = [h.herd for h in obj.herds.all()]
new_herds = set(herds.keys()).difference(existing_herds)
old_herds = set(existing_herds).difference(herds.keys())
existing_maintainers = [m.email for m in obj.maintainers.all()]
new_maintainers = set(
maintainers.keys()).difference(existing_maintainers
)
old_maintainers = set(
existing_maintainers).difference(maintainers.keys()
)
for herd in obj.herds.all():
if herd.herd in old_herds:
obj.herds.remove(herd)
for herd in new_herds:
herd = self.store_herd(*herds[herd])
obj.herds.add(herd)
for maintainer in obj.maintainers.all():
if maintainer.email in old_maintainers:
obj.maintainers.remove(maintainer)
for maintainer in new_maintainers:
maintainer = maintainers[maintainer]
try:
maintainer = self.store_maintainer(
maintainer.name, maintainer.email
)
obj.maintainers.add(maintainer)
except ValidationError:
self.logger.error(
self.style.ERROR("Bad maintainer: '%s' '%s'" % \
(maintainer.name, maintainer.email))
)
obj.save()
return True
def store_herd(self, name, email):
if not name:
name = '{nil}'
name = name.strip("\r").strip("\n").strip("\t").strip()
herd, created = Herd.objects.get_or_create(
herd=name,
defaults={"email": email}
)
if created:
self.logger.info('+ [h] %s <%s>' % (name, email))
herd.email = email
herd.save()
return herd
def store_maintainer(self, name, email):
if not name:
name = email
if not name:
name = '{nil}'
maintainer, created = Maintainer.objects.get_or_create(
email=email,
defaults={"name": name}
)
if created:
self.logger.info(
'+ [m] %s <%s>' % (name.encode('utf-8'), email)
)
return maintainer
def scan_metadata(packages=None, logger=None):
scan_handler = ScanMetadata(logger=logger)
if not packages:
packages = Package.objects.all()
for pkg in packages:
if isinstance(pkg, Package):
result = scan_handler.scan('%s/%s' % (pkg.category, pkg.name), pkg)
else:
result = scan_handler.scan(pkg)
return result

View File

@ -0,0 +1,307 @@
import subprocess
import portage
import os
import re
from xml.dom.minidom import parseString
from django.db.transaction import commit_on_success
from django.core.management.color import color_style
from euscan.helpers import get_version_type
from djeuscan.processing import FakeLogger
from djeuscan.models import Package, Version, VersionLog
class ScanPortage(object):
def __init__(self, logger=None, no_log=False, purge_packages=False,
kill_versions=False):
self.logger = logger or FakeLogger()
self.no_log = no_log
self.purge_packages = purge_packages
self.kill_versions = kill_versions
self.style = color_style()
self._cache = {'packages': {}, 'versions': {}}
self._overlays = None
def cache_hash_package(self, category, name):
return '%s/%s' % (category, name)
def cache_store_package(self, package):
key = self.cache_hash_package(package.category, package.name)
self._cache['packages'][key] = package
def cache_get_package(self, category, name):
return self._cache['packages'].get(
self.cache_hash_package(category, name)
)
def cache_hash_version(self, category, name, version, revision, slot,
overlay):
key = '%s/%s-%s-r%s %s %s' % (category, name,
version, revision,
slot, overlay)
return key
def cache_get_version(self, category, name, version, revision, slot,
overlay):
key = self.cache_hash_version(category, name, version, revision, slot,
overlay)
return self._cache['versions'].get(key)
def cache_store_version(self, version):
key = self.cache_hash_version(
version.package.category, version.package.name, version.version,
version.revision, version.slot, version.overlay
)
self._cache['versions'][key] = version
def overlays(self):
if self._overlays:
return self._overlays
env = os.environ
env['OVERLAYS_LIST'] = 'all'
env['PRINT_COUNT_ALWAYS'] = 'never'
cmd = ['eix', '-!']
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
output = output.strip().strip('\n').split('\n')
overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
self._overlays = {}
for line in output:
match = overlay_re.match(line)
if not match:
continue
self._overlays[match.group('key')] = match.group('name')
return self._overlays
def scan(self, query=None):
if self.purge_packages:
with commit_on_success():
for package in Package.objects.all():
self.logger.info('- [p] %s' % (package))
package.delete()
cmd = ['eix', '--xml', '--pure-packages', '-x']
if query:
cmd.extend(['--exact', query])
if self.kill_versions:
self.logger.info('Killing existing versions...')
Version.objects.filter(packaged=True).update(alive=False)
self.logger.info('done')
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).\
communicate()[0]
if len(output) == 0:
if not query:
return
if self.purge_packages:
self.logger.info('- [p] %s' % (query))
if '/' in query:
cat, pkg = portage.catsplit(query)
Package.objects.filter(category=cat, name=pkg).delete()
else:
Package.objects.filter(name=query).delete()
else:
self.logger.error(
self.style.ERROR(
"Unknown package '%s'" % query
)
)
return
dom = parseString(output)
for category_tag in dom.getElementsByTagName("category"):
for package_tag in category_tag.getElementsByTagName("package"):
cat = category_tag.getAttribute("name")
pkg = package_tag.getAttribute("name")
homepage_tags = package_tag.getElementsByTagName("homepage")
try:
homepage = homepage_tags[0].firstChild.nodeValue
except (IndexError, AttributeError):
homepage = ""
desc_tags = package_tag.getElementsByTagName("description")
try:
desc = desc_tags[0].firstChild.nodeValue
except (IndexError, AttributeError):
desc = ""
with commit_on_success():
package = self.store_package(cat, pkg, homepage, desc)
for version_tag in package_tag.\
getElementsByTagName("version"):
cpv = "%s/%s-%s" % (cat, pkg,
version_tag.getAttribute("id"))
slot = version_tag.getAttribute("slot")
overlay = version_tag.getAttribute("overlay")
self.store_version(package, cpv, slot, overlay)
def store_package(self, cat, pkg, homepage, description):
created = False
obj = self.cache_get_package(cat, pkg)
if not obj:
obj, created = Package.objects.get_or_create(
category=cat,
name=pkg,
homepage=homepage,
description=description,
)
self.cache_store_package(obj)
if created:
self.logger.info('+ [p] %s/%s' % (cat, pkg))
# Set all versions dead, then set found versions alive and
# delete old versions
if not self.kill_versions:
Version.objects.filter(
package=obj,
packaged=True
).update(alive=False)
return obj
def store_version(self, package, cpv, slot, overlay):
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
overlays = self.overlays()
if overlay in overlays:
overlay = overlays[overlay]
else:
overlay = 'gentoo'
created = False
obj = self.cache_get_version(
package.category, package.name, ver, rev, slot, overlay
)
if not obj:
obj, created = Version.objects.get_or_create(
package=package, slot=slot,
revision=rev, version=ver,
overlay=overlay,
defaults={
"alive": True,
"packaged": True,
"version_type": get_version_type(ver),
"confidence": 100,
"handler": "portage"
}
)
if not created: # Created objects have defaults values
obj.alive = True
obj.packaged = True
obj.save()
if created:
self.cache_store_version(obj)
# nothing to do (note: it can't be an upstream version because
# overlay can't be empty here)
if not created:
return
self.logger.info('+ [v] %s' % (obj))
if overlay == 'gentoo':
package.n_packaged += 1
else:
package.n_overlay += 1
package.n_versions += 1
package.save()
if self.no_log:
return
VersionLog.objects.create(
package=obj.package,
action=VersionLog.VERSION_ADDED,
slot=obj.slot,
revision=obj.revision,
version=obj.version,
overlay=obj.overlay
)
@commit_on_success
def do_purge_versions(logger=None, no_log=False):
logger = logger or FakeLogger()
# For each dead versions
for version in Version.objects.filter(packaged=True, alive=False):
if version.overlay == 'gentoo':
version.package.n_packaged -= 1
else:
version.package.n_overlay -= 1
version.package.n_versions -= 1
version.package.save()
logger.info('- [v] %s' % (version))
if no_log:
continue
VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED,
slot=version.slot,
revision=version.revision,
version=version.version,
overlay=version.overlay
)
Version.objects.filter(packaged=True, alive=False).delete()
def scan_portage(packages=None, no_log=False, purge_packages=False,
purge_versions=False, prefetch=False, logger=None):
logger = logger or FakeLogger()
kill_versions = False
if packages is None:
prefetch = True
kill_versions = True
scan_handler = ScanPortage(
logger=logger,
no_log=no_log,
purge_packages=purge_packages,
kill_versions=kill_versions,
)
logger.info('Scanning portage tree...')
if prefetch:
logger.info('Prefetching objects...')
for package in Package.objects.all():
scan_handler.cache_store_package(package)
for version in Version.objects.select_related('package').all():
scan_handler.cache_store_version(version)
logger.info('done')
if not packages:
scan_handler.scan()
else:
for pkg in packages:
if isinstance(pkg, Package):
scan_handler.scan('%s/%s' % (pkg.category, pkg.name))
else:
scan_handler.scan(pkg)
if purge_versions:
do_purge_versions(logger=logger, no_log=no_log)
logger.info('Done.')
return True

View File

@ -0,0 +1,161 @@
import portage
from django.utils import timezone
from django.db.transaction import commit_on_success
from euscan import CONFIG, output
from euscan.scan import scan_upstream as euscan_scan_upstream
from djeuscan.processing import FakeLogger
from djeuscan.models import Package, Version, EuscanResult, VersionLog
class ScanUpstream(object):
def __init__(self, logger=None):
self.logger = logger or FakeLogger()
def scan(self, package):
CONFIG["format"] = "dict"
output.clean()
output.set_query(package)
euscan_scan_upstream(package)
out = output.get_formatted_output()
out_json = output.get_formatted_output("json")
try:
cpv = out[package]["metadata"]["cpv"]
scan_time = out[package]["metadata"]["scan_time"]
ebuild = out[package]["metadata"]["ebuild"]
except KeyError:
return {}
with commit_on_success():
obj = self.store_package(cpv)
for res in out[package]["result"]:
self.store_version(
obj,
res["version"],
" ".join(res["urls"]),
res["type"],
res["handler"],
res["confidence"],
)
self.store_result(obj, out_json, scan_time, ebuild)
return out
def store_result(self, package, formatted_log, scan_time, ebuild):
# Remove previous logs
EuscanResult.objects.filter(package=package).delete()
obj = EuscanResult()
obj.package = package
obj.result = formatted_log
obj.datetime = timezone.now()
obj.scan_time = scan_time
obj.ebuild = ebuild
obj.save()
def store_package(self, cpv):
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
if created:
self.logger.info('+ [p] %s/%s' % (cat, pkg))
# Set all versions dead, then set found versions alive and
# delete old versions
Version.objects.filter(package=obj, packaged=False).update(alive=False)
return obj
def store_version(self, package, ver, url, version_type, handler,
confidence):
obj, created = Version.objects.get_or_create(
package=package,
slot='',
revision='r0',
version=ver,
overlay='',
defaults={"alive": True, "urls": url, "packaged": False,
"version_type": version_type, "handler": handler,
"confidence": confidence}
)
if not created:
obj.alive = True
obj.urls = url
obj.packaged = False
obj.save()
# If it's not a new version, just update the object and continue
if not created:
return
self.logger.info('+ [u] %s %s' % (obj, url))
VersionLog.objects.create(
package=package,
action=VersionLog.VERSION_ADDED,
slot='',
revision='r0',
version=ver,
overlay=''
)
package.n_versions += 1
package.save()
@commit_on_success
def do_purge_versions(logger=None):
logger = logger or FakeLogger()
# For each dead versions
for version in Version.objects.filter(packaged=False, alive=False):
VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED,
slot=version.slot,
revision=version.revision,
version=version.version,
overlay=version.overlay
)
version.package.n_versions -= 1
version.package.save()
logger.info('- [u] %s %s' % (version, version.urls))
Version.objects.filter(packaged=False, alive=False).delete()
def scan_upstream(packages=None, purge_versions=False,
logger=None):
logger = logger or FakeLogger()
scan_handler = ScanUpstream(logger=logger)
logger.info('Scanning upstream...')
if not packages:
packages = Package.objects.all()
result = True
for pkg in packages:
if isinstance(pkg, Package):
curr = scan_handler.scan('%s/%s' % (pkg.category, pkg.name))
else:
curr = scan_handler.scan(pkg)
if not curr:
result = False
if purge_versions:
do_purge_versions(logger=logger)
logger.info('Done.')
return result

View File

@ -0,0 +1,175 @@
from django.db.transaction import commit_on_success
from django.utils import timezone
from djeuscan.models import Package, Herd, Maintainer, Version
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from djeuscan import charts
from djeuscan.processing import FakeLogger
from distutils.version import StrictVersion, LooseVersion
def _compare_versions(version1, version2):
try:
return cmp(StrictVersion(version1), StrictVersion(version2))
# in case of abnormal version number, fall back to LooseVersion
except ValueError:
return cmp(LooseVersion(version1), LooseVersion(version2))
def _add_safe(storage, key):
if key not in storage:
storage[key] = 1
else:
storage[key] += 1
def _add_last_ver(storage, version):
key = version['package_id']
if key not in storage:
storage[key] = version
return
if version['version'].startswith('9999'):
return
if _compare_versions(storage[key]['version'],
version['version']) < 0:
storage[key] = version
@commit_on_success
def update_counters(fast=False, nolog=False, logger=None):
logger = logger or FakeLogger()
now = timezone.now()
categories = {}
herds = {}
maintainers = {}
wlog = None
if not nolog:
wlog = WorldLog()
wlog.datetime = now
for cat in Package.objects.values('category').distinct():
clog = CategoryLog()
clog.datetime = now
clog.category = cat['category']
categories[clog.category] = clog
for herd in Herd.objects.all():
hlog = HerdLog()
hlog.datetime = now
hlog.herd = herd
herds[herd.id] = hlog
for maintainer in Maintainer.objects.all():
mlog = MaintainerLog()
mlog.datetime = now
mlog.maintainer = maintainer
maintainers[maintainer.id] = mlog
package_queryset = Package.objects.all()
n_versions = {}
n_packaged = {}
n_overlay = {}
last_versions_gentoo = {}
last_versions_overlay = {}
last_versions_upstream = {}
if not fast:
attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
for version in Version.objects.all().values(*attrs):
overlay, packaged = version['overlay'], version['packaged']
package_id = version['package_id']
_add_safe(n_versions, package_id)
if not packaged:
_add_last_ver(last_versions_upstream, version)
continue
if overlay == 'gentoo':
_add_safe(n_packaged, package_id)
_add_last_ver(last_versions_gentoo, version)
else:
_add_safe(n_overlay, package_id)
_add_last_ver(last_versions_overlay, version)
for package in package_queryset.select_related('herds', 'maintainers'):
if not fast:
package.n_versions = n_versions.get(package.id, 0)
package.n_packaged = n_packaged.get(package.id, 0)
package.n_overlay = n_overlay.get(package.id, 0)
default = {'id': None}
package.last_version_gentoo_id = last_versions_gentoo.get(
package.id, default
)['id']
package.last_version_overlay_id = last_versions_overlay.get(
package.id, default
)['id']
package.last_version_upstream_id = last_versions_upstream.get(
package.id, default
)['id']
package.save()
n_packages_gentoo = int(package.n_packaged == package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged \
+ package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay \
< package.n_versions)
def update_row(storage, key):
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_outdated += n_packages_outdated
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
def update_log(storage, qs):
for row in qs:
update_row(storage, row['id'])
if not nolog:
update_log(herds, package.herds.all().values('id'))
update_log(maintainers, package.maintainers.all().values('id'))
update_row(categories, package.category)
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
if nolog:
return
for clog in categories.values():
logger.info('+ [cl] %s\n' % clog)
charts.rrd_update('category-%s' % clog.category, now, clog)
clog.save()
for hlog in herds.values():
logger.info('+ [hl] %s\n' % hlog)
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
hlog.save()
for mlog in maintainers.values():
logger.info('+ [ml] %s\n' % mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save()
charts.rrd_update('world', now, wlog)
wlog.save()

View File

@ -0,0 +1,66 @@
import subprocess
from StringIO import StringIO
from django.conf import settings
from djeuscan.processing import FakeLogger
def _launch_command(cmd):
"""
Helper for launching shell commands inside tasks
"""
fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = StringIO(fp.communicate()[0])
return output.getvalue()
def emerge_sync():
"""
Launches an emerge --sync
"""
cmd = ["emerge", "--sync", "--root", settings.PORTAGE_ROOT,
"--config-root", settings.PORTAGE_CONFIGROOT]
return _launch_command(cmd)
def layman_sync():
"""
Syncs Layman repos
"""
from layman import Layman
l = Layman(config=settings.LAYMAN_CONFIG)
return l.sync(l.get_installed(), output_results=False)
def emerge_regen():
"""
Launches emerge --regen
"""
cmd = [
"emerge", "--regen", "--jobs", settings.EMERGE_REGEN_JOBS, "--root",
settings.PORTAGE_ROOT, "--config-root", settings.PORTAGE_CONFIGROOT
]
return _launch_command(cmd)
def eix_update():
"""
Launches eix-update
"""
cmd = ["eix-update"]
return _launch_command(cmd)
def update_portage_trees(logger=None):
logger = logger or FakeLogger()
logger.info("Running emerge --sync")
emerge_sync()
logger.info("Running layman --sync")
layman_sync()
logger.info("Running emerge --regen")
emerge_regen()
logger.info("Running eix-update")
eix_update()
logger.info("Done!")

292
euscanwww/djeuscan/tasks.py Normal file
View File

@ -0,0 +1,292 @@
"""
Celery tasks for djeuscan
"""
from itertools import islice
from celery.task import task, periodic_task
from celery.task.schedules import crontab
from celery.task.sets import TaskSet
from django.conf import settings
from djeuscan.models import Package, RefreshPackageQuery
from djeuscan.processing.regen_rrds import regen_rrds
from djeuscan.processing.update_counters import update_counters
from djeuscan.processing.scan_metadata import scan_metadata
from djeuscan.processing.scan_portage import scan_portage
from djeuscan.processing.scan_upstream import scan_upstream
from djeuscan.processing.update_portage_trees import update_portage_trees
class TaskFailedException(Exception):
"""
Exception for failed tasks
"""
pass
def _chunks(it, n):
"""
Chunk generator, takes an iterator and the desired size of the chunk
"""
for first in it:
yield [first] + list(islice(it, n - 1))
def _run_in_chunks(task, packages, kwargs=None,
concurrently=settings.TASKS_CONCURRENTLY,
n=settings.TASKS_SUBTASK_PACKAGES):
"""
Launches a TaskSet at a time with <concurrently> subtasks.
Each subtask has <n> packages to handle
"""
output = []
chunk_generator = _chunks(iter(packages), n)
done = False
while not done:
tasks = []
for _ in range(concurrently):
try:
chunk = chunk_generator.next()
except StopIteration:
done = True
else:
tasks.append(task.subtask((chunk, ), kwargs))
job = TaskSet(tasks=tasks)
result = job.apply_async()
# TODO: understand why this causes timeout
output.extend(list(result.join()))
return output
@task
def regen_rrds_task():
"""
Regenerate RRDs
"""
return regen_rrds()
@task
def update_counters_task(fast=True):
"""
Updates counters
"""
return update_counters(fast=fast)
@task
def _scan_metadata_task(packages):
"""
Scans metadata for the given set of packages
"""
logger = _scan_metadata_task.get_logger()
logger.info("Starting metadata scanning subtask for %d packages...",
len(packages))
result = scan_metadata(
packages=packages,
logger=logger,
)
if not result:
raise TaskFailedException
return result
@task
def scan_metadata_list_task(query):
"""
Runs a parallel metadata scan for packages in the query list (space
separated string). Task used only from the web interface.
"""
return _run_in_chunks(_scan_metadata_task, [p for p in query.split()])
@task
def scan_metadata_all_task():
"""
Runs a parallel metadata scan for all packages
"""
return _run_in_chunks(_scan_metadata_task, Package.objects.all())
@task
def _scan_portage_task(packages, no_log=False, purge_packages=False,
purge_versions=False, prefetch=False):
"""
Scans portage for the given set of packages
"""
logger = _scan_portage_task.get_logger()
if packages:
logger.info("Starting portage scanning subtask for %d packages...",
len(packages))
else:
logger.info("Starting portage scanning for all packages...")
result = scan_portage(
packages=packages,
no_log=no_log,
purge_packages=purge_packages,
purge_versions=purge_versions,
prefetch=prefetch,
logger=logger,
)
if not result:
raise TaskFailedException
return result
@task
def scan_portage_list_task(query, no_log=False, purge_packages=False,
purge_versions=False, prefetch=False):
"""
Runs a parallel portage scan for packages in the query list (space
separated string). Task used only from the web interface.
"""
kwargs = {"no_log": no_log, "purge_packages": purge_packages,
"purge_versions": purge_versions, "prefetch": prefetch}
return _run_in_chunks(_scan_portage_task, [p for p in query.split()],
kwargs)
@task
def scan_portage_all_task(no_log=False, purge_packages=False,
purge_versions=False, prefetch=False):
"""
Runs a syncronous portage scan for all packages
"""
return _scan_portage_task(
packages=None,
no_log=no_log,
purge_packages=purge_packages,
purge_versions=purge_versions,
prefetch=prefetch,
)
@task
def _scan_upstream_task(packages, purge_versions=False):
"""
Scans upstream for the given set of packages
"""
logger = _scan_upstream_task.get_logger()
logger.info("Starting upstream scanning subtask for %d packages...",
len(packages))
result = scan_upstream(
packages=packages,
purge_versions=purge_versions,
logger=logger,
)
if not result:
raise TaskFailedException
return result
@task
def scan_upstream_list_task(query, purge_versions=False):
"""
Runs a parallel upstream scan for packages in the query list (space
separated string). Task used only from the web interface.
"""
kwargs = {"purge_versions": purge_versions}
return _run_in_chunks(_scan_upstream_task, [p for p in query.split()],
kwargs)
@task
def scan_upstream_all_task(purge_versions=False):
"""
Runs a parallel portage scan for all packages
"""
kwargs = {"purge_versions": purge_versions}
return _run_in_chunks(_scan_upstream_task, Package.objects.all(), kwargs)
@task
def update_portage_trees_task():
"""
Update portage tree
"""
logger = update_portage_trees_task.get_logger()
update_portage_trees(logger=logger)
@task
def update_task(update_portage_trees=True, scan_portage=True,
scan_metadata=True, scan_upstream=True, update_counter=True):
"""
Update the whole euscan system
"""
if update_portage_trees:
update_portage_trees_task()
if scan_portage:
scan_portage_all_task(prefetch=True, purge_packages=True,
purge_versions=True)
# metadata and upstream scan can run concurrently, launch them
# asynchronously and wait for them to finish
metadata_job = None
if scan_metadata:
metadata_job = scan_metadata_all_task().delay()
upstream_job = None
if scan_upstream:
upstream_job = scan_upstream_all_task().delay()
if metadata_job:
metadata_job.wait()
if upstream_job:
upstream_job.wait()
update_counters(fast=False)
@task
def scan_package_task(package):
_scan_portage_task([package], purge_packages=True, purge_versions=True)
_scan_metadata_task([package])
_scan_upstream_task([package])
@periodic_task(run_every=crontab(minute="*/1"))
def consume_refresh_package_request():
"""
Satisfies user requests for package refreshing, runs every minute
"""
try:
obj = RefreshPackageQuery.objects.all().order_by('-priority')[0]
except IndexError:
return {}
else:
result = scan_package_task(obj.query)
obj.delete()
return result
@periodic_task(run_every=crontab(hour=03, minute=00, day_of_week=1))
def update_periodic_task():
"""
Runs a whole update once a week
"""
update_task()
admin_tasks = [
regen_rrds_task,
update_counters_task,
scan_metadata_list_task,
scan_metadata_all_task,
scan_portage_all_task,
scan_portage_list_task,
scan_upstream_all_task,
scan_upstream_list_task,
update_portage_trees_task,
update_task,
scan_package_task,
]

View File

@ -10,6 +10,7 @@
<link rel="alternate" type="application/atom+xml" title="Global log" href="{% url "global_feed" %}" /> <link rel="alternate" type="application/atom+xml" title="Global log" href="{% url "global_feed" %}" />
{% endblock %} {% endblock %}
{% block css %} {% block css %}
<link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/bootstrap.min.css" media="screen" title="Normal" />
<link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/style.css" media="screen" title="Normal" /> <link rel="stylesheet" type="text/css" href="{{ STATIC_URL }}css/style.css" media="screen" title="Normal" />
{% endblock %} {% endblock %}
{% block javascript %} {% block javascript %}
@ -22,12 +23,20 @@
</a> </a>
{% block header %}<h1>Ebuild Upstream Scanner (euscan)</h1>{% endblock %} {% block header %}<h1>Ebuild Upstream Scanner (euscan)</h1>{% endblock %}
</div> </div>
<div id="content"> <div class="row-fluid">
<div class="span10">
<div class="row-fluid">
<div class="span1"></div>
<div id="content" class="rounded span11">
{% block content %}{% endblock %} {% block content %}{% endblock %}
</div> </div>
<div id="menus"> </div>
</div>
<div class="span2">
<div class="row-fluid">
<div id="menus" class="span11">
{% block menus %} {% block menus %}
<div class="menu"> <div class="menu rounded">
<ul> <ul>
{% block menu %} {% block menu %}
<li><a href="{% url "index" %}">Home</a></li> <li><a href="{% url "index" %}">Home</a></li>
@ -37,12 +46,25 @@
<li><a href="{% url "overlays" %}">Overlays</a></li> <li><a href="{% url "overlays" %}">Overlays</a></li>
<li><a href="{% url "world" %}">Scan World</a></li> <li><a href="{% url "world" %}">Scan World</a></li>
<li><a href="{% url "statistics" %}">Statistics</a></li> <li><a href="{% url "statistics" %}">Statistics</a></li>
<!--
<li>---</li> <li>---</li>
<li><a href="#">Login</a></li>
<li><a href="#">Register</a></li> {% if user.is_authenticated %}
--> <li><a href="{% url "accounts_index" %}">{{ user }}'s profile</a></li>
<ul class="submenu">
<li><a href="{% url "accounts_categories" %}">Categories</a></li>
<li><a href="{% url "accounts_herds" %}">Herds</a></li>
<li><a href="{% url "accounts_maintainers" %}">Maintainers</a></li>
<li><a href="{% url "accounts_packages" %}">Packages</a></li>
</ul>
<li><a href="{% url "django.contrib.auth.views.logout" %}">Logout</a></li>
{% else %}
<li><a href="{% url "django.contrib.auth.views.login" %}?next={% url "accounts_index" %}">Login</a></li>
<li><a href="{% url "registration.views.register" %}">Register</a></li>
{% endif %}
<li>---</li> <li>---</li>
{% block menu_feed %} {% block menu_feed %}
<li> <li>
<img src="{{ STATIC_URL }}img/feed.png" alt="feed" /> <img src="{{ STATIC_URL }}img/feed.png" alt="feed" />
@ -57,6 +79,10 @@
</div> </div>
{% endblock %} {% endblock %}
</div> </div>
<div class="span1"></div>
</div>
</div>
</div>
<div id="footer"> <div id="footer">
<p> <p>
{% block last_scan %} {% block last_scan %}

View File

@ -0,0 +1,81 @@
{% extends "admin/change_list.html" %}
{% load url from future %}
{% block object-tools %}
<div>
<span id="task_selector"></span>
<form id="task_data"></form>
<a href="#" id="task_launch">Launch task</a>
</div>
<script type="text/javascript">
(function($){
$(document).ready(function() {
$.get("{% url "registered_tasks" %}", function (data) {
var selector = $("<select/>");
selector.append(
$("<option/>").text("-------------").attr("selected", "selected")
);
$.each(data.tasks, function(task) {
var t = data["tasks"][task];
selector.append(
$("<option/>").val(task).text(task)
)
});
selector.change(function() {
var task = data["tasks"][$(this).val()];
$("#task_data").empty();
if (task.args) {
$.each(task.args, function(i) {
var arg = task.args[i]
, arg_input = $("<input/>").attr("name", arg)
, default_arg;
if (task.defaults) {
var default_arg_i = task.args.length - 1 - i;
if (task.defaults[default_arg_i] !== undefined) {
default_arg = task.defaults[default_arg_i];
arg_input.val(default_arg);
if (task.defaults_types && task.defaults_types[default_arg_i] === "bool") {
arg_input.attr("type", "checkbox").attr("checked", default_arg).val("true");
}
else {
return true;
}
}
}
$("#task_data").append(
$("<label/>").text(arg + ": ").attr("for", arg)
).append(
arg_input
);
});
}
})
$("#task_selector").append(selector);
});
$("#task_launch").click(function() {
var task_name = $("#task_selector").find("select").val();
var data = $("#task_data").serialize();
var url = "{% url "apply_task" "task_name" %}";
$.post(url.replace("task_name", task_name), data, function() {
alert("Submitted!");
location.reload();
});
});
setTimeout(location.reload, 30000);
});
})(django.jQuery);
</script>
{% endblock %}

View File

@ -0,0 +1,32 @@
{% load url from future %}
{% load euscan %}
<table id="table" class="display">
<thead>
<th>Category</th>
<th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
<th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
<th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
<th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
{% if extras %}
<th>Graphs</th>
{% endif %}
</thead>
<tbody>
{% for category in categories %}
<tr>
<td>
<a href="{% url "category" category.category %}">{{ category.category }}</a>
{% package_bar category %}
</td>
{% package_cols category %}
{% if extras %}
<td>
<img src="{% url "chart_category" category.category 'packages-monthly-small' %}" />
<img src="{% url "chart_category" category.category 'versions-monthly-small' %}" />
</td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>

View File

@ -24,7 +24,7 @@
{% block menus %} {% block menus %}
{{ block.super }} {{ block.super }}
<div class="menu"> <div class="menu notfirst-menu rounded">
<ul> <ul>
<li><img src="{{ STATIC_URL }}img/gentoo-icon.png" /> Gentoo</li> <li><img src="{{ STATIC_URL }}img/gentoo-icon.png" /> Gentoo</li>
<li><img src="{{ STATIC_URL }}img/overlay-icon.png" /> Overlays</li> <li><img src="{{ STATIC_URL }}img/overlay-icon.png" /> Overlays</li>

View File

@ -0,0 +1,34 @@
{% load url from future %}
{% load euscan %}
<table id="table" class="display">
<thead>
<th>Herd</th>
<th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
<th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
<th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
<th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
{% if extras %}
<th>Graphs</th>
{% endif %}
</thead>
<tbody>
{% for herd in herds %}
<tr>
<td>
<a href="{% url "herd" herd.herds__herd %}">
{{ herd.herds__herd }}
</a>
{% package_bar herd %}
</td>
{% package_cols herd %}
{% if extras %}
<td>
<img src="{% url "chart_herd" herd.herds__herd 'packages-monthly-small' %}" />
<img src="{% url "chart_herd" herd.herds__herd 'versions-monthly-small' %}" />
</td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>

View File

@ -0,0 +1,40 @@
{% load url from future %}
{% load euscan %}
<table id="table" class="display">
<thead>
<th>Maintainer</th>
<th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
<th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
<th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
<th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
{% if extras %}
<th>Graphs</th>
{% endif %}
</thead>
<tbody>
{% for maintainer in maintainers %}
<tr>
<td>
<a href="{% url "maintainer" maintainer.maintainers__id %}">
{% if maintainer.maintainers__name != maintainer.maintainers__email %}
{{ maintainer.maintainers__name }} &lt;{{ maintainer.maintainers__email }}&gt;
{% else %}
{{ maintainer.maintainers__name }}
{% endif %}
</a>
{% package_bar maintainer %}
</td>
{% package_cols maintainer %}
{% if extras %}
<td>
<img src="{% url "chart_maintainer" maintainer.maintainers__id 'packages-monthly-small' %}" />
<img src="{% url "chart_maintainer" maintainer.maintainers__id 'versions-monthly-small' %}" />
</td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>

View File

@ -1,4 +1,4 @@
{% load packages %} {% load euscan %}
{% load mul %} {% load mul %}
{% load sub %} {% load sub %}
{% load div %} {% load div %}

View File

@ -1,4 +1,4 @@
{% load packages %} {% load euscan %}
{% load sub %} {% load sub %}
{% load div %} {% load div %}
{% load mul %} {% load mul %}

View File

@ -0,0 +1,15 @@
{% extends "euscan/_datatable.html" %}
{% load url from future %}
{% load euscan %}
{% block title %}
{{ block.super }} - Watched categories
{% endblock %}
{% block content %}
<h2>Watched categories</h2>
{% categories_table categories %}
{% endblock %}

View File

@ -0,0 +1,14 @@
{% extends "euscan/_datatable.html" %}
{% load url from future %}
{% load euscan %}
{% block title %}
{{ block.super }} - Watched herds
{% endblock %}
{% block content %}
<h2>Watched herds</h2>
{% herds_table herds %}
{% endblock %}

View File

@ -0,0 +1,11 @@
{% extends "_base.html" %}
{% load url from future %}
{% block title %}
{{ block.super }} - Welcome {{ user }}
{% endblock %}
{% block content %}
<h2>Welcome {{ user }}</h2>
{% endblock %}

View File

@ -0,0 +1,14 @@
{% extends "euscan/_datatable.html" %}
{% load url from future %}
{% load euscan %}
{% block title %}
{{ block.super }} - Watched maintainers
{% endblock %}
{% block content %}
<h2>Watched maintainers</h2>
{% maintainers_table maintainers %}
{% endblock %}

View File

@ -0,0 +1,16 @@
{% extends "euscan/_datatable.html" %}
{% load url from future %}
{% load euscan %}
{% block title %}
{{ block.super }} - Watched packages
{% endblock %}
{% block content %}
<h2>Watched packages</h2>
{% packages packages %}
{% endblock %}

View File

@ -1,6 +1,6 @@
{% extends "euscan/_datatable.html" %} {% extends "euscan/_datatable.html" %}
{% load packages %} {% load euscan %}
{% load url from future %} {% load url from future %}
{% block title %} {% block title %}
@ -10,33 +10,6 @@
{% block content %} {% block content %}
<h2>Categories</h2> <h2>Categories</h2>
<table id="table" class="display"> {% categories_table categories request.GET.extras %}
<thead>
<th>Category</th>
<th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
<th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
<th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
<th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
{% if request.GET.extras %}
<th>Graphs</th>
{% endif %}
</thead>
<tbody>
{% for category in categories %}
<tr>
<td>
<a href="{% url "category" category.category %}">{{ category.category }}</a>
{% package_bar category %}
</td>
{% package_cols category %}
{% if request.GET.extras %}
<td>
<img src="{% url "chart_category" category.category 'packages-monthly-small' %}" />
<img src="{% url "chart_category" category.category 'versions-monthly-small' %}" />
</td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %} {% endblock %}

View File

@ -1,6 +1,6 @@
{% extends "euscan/_datatable.html" %} {% extends "euscan/_datatable.html" %}
{% load packages %} {% load euscan %}
{% load url from future %} {% load url from future %}
{% block title %} {% block title %}

View File

@ -1,6 +1,6 @@
{% extends "euscan/_datatable.html" %} {% extends "euscan/_datatable.html" %}
{% load packages %} {% load euscan %}
{% load url from future %} {% load url from future %}
{% block title %} {% block title %}

View File

@ -1,6 +1,6 @@
{% extends "euscan/_datatable.html" %} {% extends "euscan/_datatable.html" %}
{% load packages %} {% load euscan %}
{% load url from future %} {% load url from future %}
{% block title %} {% block title %}
@ -9,35 +9,7 @@
{% block content %} {% block content %}
<h2>Herds</h2> <h2>Herds</h2>
<table id="table" class="display">
<thead> {% herds_table herds request.GET.extras %}
<th>Herd</th>
<th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
<th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
<th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
<th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
{% if request.GET.extras %}
<th>Graphs</th>
{% endif %}
</thead>
<tbody>
{% for herd in herds %}
<tr>
<td>
<a href="{% url "herd" herd.herds__herd %}">
{{ herd.herds__herd }}
</a>
{% package_bar herd %}
</td>
{% package_cols herd %}
{% if request.GET.extras %}
<td>
<img src="{% url "chart_herd" herd.herds__herd 'packages-monthly-small' %}" />
<img src="{% url "chart_herd" herd.herds__herd 'versions-monthly-small' %}" />
</td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %} {% endblock %}

View File

@ -1,6 +1,6 @@
{% extends "euscan/_datatable.html" %} {% extends "euscan/_datatable.html" %}
{% load packages %} {% load euscan %}
{% load url from future %} {% load url from future %}
{% block title %} {% block title %}

View File

@ -1,6 +1,6 @@
{% extends "euscan/_datatable.html" %} {% extends "euscan/_datatable.html" %}
{% load packages %} {% load euscan %}
{% load url from future %} {% load url from future %}
{% block title %} {% block title %}
@ -9,41 +9,7 @@
{% block content %} {% block content %}
<h2>Maintainers</h2> <h2>Maintainers</h2>
<table id="table" class="display">
<thead>
<th>Maintainer</th>
<th><img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="Versions in Gentoo" /></th>
<th><img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="Versions in Overlays" /></th>
<th><img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Versions only upstream" /></th>
<th><img src="{{ STATIC_URL }}img/freshness-icon.png" title="Freshness" /></th>
{% if request.GET.extras %}
<th>Graphs</th>
{% endif %}
</thead>
<tbody>
{% for maintainer in maintainers %}
<tr>
<td>
<a href="{% url "maintainer" maintainer.maintainers__id %}">
{% if maintainer.maintainers__name != maintainer.maintainers__email %}
{{ maintainer.maintainers__name }} &lt;{{ maintainer.maintainers__email }}&gt;
{% else %}
{{ maintainer.maintainers__name }}
{% endif %}
</a>
{% package_bar maintainer %}
</td>
{% package_cols maintainer %}
{% if request.GET.extras %} {% maintainers_table maintainers request.GET.extras %}
<td>
<img src="{% url "chart_maintainer" maintainer.maintainers__id 'packages-monthly-small' %}" />
<img src="{% url "chart_maintainer" maintainer.maintainers__id 'versions-monthly-small' %}" />
</td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %} {% endblock %}

View File

@ -1,6 +1,6 @@
{% extends "euscan/_datatable.html" %} {% extends "euscan/_datatable.html" %}
{% load packages %} {% load euscan %}
{% block title %} {% block title %}
{{ block.super }} - Overlay: {{ overlay }} {{ block.super }} - Overlay: {{ overlay }}

View File

@ -1,6 +1,6 @@
{% extends "euscan/_datatable.html" %} {% extends "euscan/_datatable.html" %}
{% load packages %} {% load euscan %}
{% load url from future %} {% load url from future %}
{% block title %} {% block title %}

View File

@ -24,7 +24,21 @@
{% endblock %} {% endblock %}
{% block content %} {% block content %}
<h2>{{ package.category }}/{{ package.name }}</h2> <h2>
{{ package.category }}/{{ package.name }}
{% if user.is_authenticated %}
<button class="btn refresh-button" data-package="{{ package.category }}/{{ package.name }}">
Refresh
</button>
<button class="btn favourite-button {% if favourited %}hide{% endif %}" data-url="{% url "favourite_package" package.category package.name %}">
Add to favourites
</button>
<button class="btn unfavourite-button {% if not favourited %}hide{% endif %}" data-url="{% url "unfavourite_package" package.category package.name %}">
Remove from favourites
</button>
{% endif %}
</h2>
<dl> <dl>
{% if package.description %} {% if package.description %}
<dt>Description</dt> <dt>Description</dt>
@ -129,4 +143,28 @@
</dd> </dd>
{% endif %} {% endif %}
</dl> </dl>
<script type="text/javascript">
$(".refresh-button").click(function() {
var url = "{% url "refresh_package" "x/x" %}";
$.post(url.replace("x/x", $(this).data("package")), function() {
alert("Submitted!");
});
});
$(".favourite-button").click(function() {
$.post($(this).data("url"), function() {
$(".unfavourite-button").removeClass("hide");
$(".favourite-button").addClass("hide");
});
});
$(".unfavourite-button").click(function() {
$.post($(this).data("url"), function() {
$(".favourite-button").removeClass("hide");
$(".unfavourite-button").addClass("hide");
});
});
</script>
{% endblock %} {% endblock %}

View File

@ -3,7 +3,6 @@
{% load url from future %} {% load url from future %}
{% block content %} {% block content %}
<h2>What's euscan ?</h2>
<h2>Statistics</h2> <h2>Statistics</h2>
<h3>Current statistics</h3> <h3>Current statistics</h3>

View File

@ -1,6 +1,6 @@
{% extends "euscan/_datatable.html" %} {% extends "euscan/_datatable.html" %}
{% load packages %} {% load euscan %}
{% block title %} {% block title %}
{{ block.super }} - World Scan {{ block.super }} - World Scan

View File

@ -0,0 +1 @@
{% extends "_base.html" %}

View File

@ -0,0 +1,13 @@
{% extends "registration/_registration_base.html" %}
{% block title %}{% if account %}Activation complete{% else %}Activation problem{% endif %}{% endblock %}
{% block content %}
{% if account %}
Thanks {{ account }}, activation complete! You may now <a href='{% url auth_login %}'>login</a>
using the username and password you set at registration.
{% else %}
Oops &ndash; it seems that your activation key is invalid.
Please check the url again.
{% endif %}
{% endblock %}

View File

@ -0,0 +1,6 @@
{% extends "registration/_registration_base.html" %}
{% block title %}Activation complete{% endblock %}
{% block content %}
Thanks, activation complete! You may now <a href='{% url auth_login %}'>login</a>
using the username and password you set at registration.
{% endblock %}

View File

@ -0,0 +1,11 @@
You (or someone pretending to be you) have asked to register an account at
{{ site.name }}. If this wasn't you, please ignore this email
and your address will be removed from our records.
To activate this account, please click the following link within the next
{{ expiration_days }} days:
http://{{site.domain}}{% url registration_activate activation_key %}
Sincerely,
{{ site.name }} Management

View File

@ -0,0 +1 @@
Account registration for {{ site.name }}

View File

@ -0,0 +1,41 @@
{% extends "registration/_registration_base.html" %}
{% load url from future %}
{% load i18n %}
{% block content %}
{% if form.errors %}
<p>Your username and password didn't match. Please try again.</p>
{% endif %}
<form method="post" action="{% url 'django.contrib.auth.views.login' %}">
{% csrf_token %}
<table>
<tr>
<th>{{ form.username.label_tag }}</th>
<td>{{ form.username }}</td>
</tr>
<tr>
<th>{{ form.password.label_tag }}</th>
<td>{{ form.password }}</td>
</tr>
<tr>
<td></td>
<td>
<input class="btn pull-right" type="submit" value="Login" />
</td>
</tr>
<tr>
<td>&nbsp;</td>
</tr>
<tr>
<td>
</td>
<td>
<a class="pull-right" href="{% url "django.contrib.auth.views.password_reset" %}">Forgot password</a>
</td>
</tr>
</table>
<input type="hidden" name="next" value="/" />
</form>
{% endblock %}

View File

@ -0,0 +1,13 @@
{% extends "registration/_registration_base.html" %}
{% load i18n %}
{% block content %}
<p>You're logged out."</p>
<p>Redirect...</p>
<script type="text/javascript">
window.location.href = "/";
</script>
{% endblock %}

View File

@ -0,0 +1,6 @@
{% extends "registration/_registration_base.html" %}
{% block title %}Password changed{% endblock %}
{% block content %}
Password successfully changed!
{% endblock %}

View File

@ -0,0 +1,16 @@
{% extends "registration/_registration_base.html" %}
{% block title %}Change password{% endblock %}
{% block content %}
<form method='post' action=''>{% csrf_token %}
<table>
{{ form }}
<tr>
<td></td>
<td>
<input type='submit' value="Change password" />
</td>
</tr>
</table>
</form>
{% endblock %}

View File

@ -0,0 +1,7 @@
{% extends "registration/_registration_base.html" %}
{% block title %}Password reset complete{% endblock %}
{% block content %}
Your password has been reset! You may now <a href="{{ login_url }}">log in</a>.
{% endblock %}

View File

@ -0,0 +1,15 @@
{% extends "registration/_registration_base.html" %}
{% block title %}Confirm password reset{% endblock %}
{% block content %}
Enter your new password below to reset your password:
<form method="post" action="">{% csrf_token %}
<table>
{{ form.as_table }}
<tr>
<td></td>
<td><input type="submit" value="Set password" /></td>
</tr>
</table>
</form>
{% endblock %}

View File

@ -0,0 +1,10 @@
{% extends "registration/_registration_base.html" %}
{% block title %}Password reset{% endblock %}
{% block content %}
<p>
We have sent you an email with a link to reset your password. Please check
your email and click the link to continue.
</p>
{% endblock %}

View File

@ -0,0 +1,16 @@
Greetings {% if user.get_full_name %}{{ user.get_full_name }}{% else %}{{ user }}{% endif %},
You are receiving this email because you (or someone pretending to be you)
requested that your password be reset on the {{ domain }} site. If you do not
wish to reset your password, please ignore this message.
To reset your password, please click the following link, or copy and paste it
into your web browser:
{{ protocol }}://{{ domain }}{% url auth_password_reset_confirm uid token %}
Your username, in case you've forgotten: {{ user.username }}
Best regards,
{{ site_name }} Management

View File

@ -0,0 +1,18 @@
{% extends "registration/_registration_base.html" %}
{% block title %}Reset password{% endblock %}
{% block content %}
Forgot your password? Enter your email in the form below and we'll send you
instructions for creating a new one.
<form method='post' action=''>{% csrf_token %}
<table>
{{ form }}
<tr>
<td></td>
<td><input type='submit' value="Reset password" /></td>
</tr>
</table>
</form>
{% endblock %}

View File

@ -0,0 +1,6 @@
{% extends "registration/_registration_base.html" %}
{% block title %}Activation email sent{% endblock %}
{% block content %}
An activation email has been sent. Please check your email and click on the link to activate your account.
{% endblock %}

View File

@ -0,0 +1,19 @@
{% extends "registration/_registration_base.html" %}
{% block title %}
Register for an account
{% endblock %}
{% block content %}
<table>
<form method='post' action=''>{% csrf_token %}
{{ form }}
<tr>
<td></td>
<td>
<input class="btn pull-right" type="submit" value="Send activation email" />
</td>
</tr>
</form>
</table>
{% endblock %}

View File

@ -0,0 +1,49 @@
from django import template
from django.conf import settings
register = template.Library()
@register.inclusion_tag('euscan/_packages.html', takes_context=True)
def packages(context, pkgs):
context['packages'] = pkgs
return context
@register.inclusion_tag('euscan/_package_cols.html', takes_context=True)
def package_cols(context, infos):
context['infos'] = infos
return context
@register.inclusion_tag('euscan/_package_bar.html', takes_context=True)
def package_bar(context, infos):
context['infos'] = infos
return context
@register.inclusion_tag('euscan/_categories_table.html')
def categories_table(categories, extras=False):
return {
"categories": categories,
"extras": extras,
"STATIC_URL": settings.STATIC_URL,
}
@register.inclusion_tag('euscan/_herds_table.html')
def herds_table(herds, extras=False):
return {
"herds": herds,
"extras": extras,
"STATIC_URL": settings.STATIC_URL,
}
@register.inclusion_tag('euscan/_maintainers_table.html')
def maintainers_table(maintainers, extras=False):
return {
"maintainers": maintainers,
"extras": extras,
"STATIC_URL": settings.STATIC_URL,
}

View File

@ -1,21 +0,0 @@
from django import template
register = template.Library()
@register.inclusion_tag('euscan/_packages.html', takes_context=True)
def packages(context, pkgs):
context['packages'] = pkgs
return context
@register.inclusion_tag('euscan/_package_cols.html', takes_context=True)
def package_cols(context, infos):
context['infos'] = infos
return context
@register.inclusion_tag('euscan/_package_bar.html', takes_context=True)
def package_bar(context, infos):
context['infos'] = infos
return context

View File

@ -1,12 +1,26 @@
from django.conf.urls.defaults import url, patterns, include from django.conf.urls.defaults import url, patterns, include
from django.contrib.auth.decorators import user_passes_test
from djcelery.views import apply as apply_task
from djeuscan.views import registered_tasks
from djeuscan.feeds import PackageFeed, CategoryFeed, HerdFeed, \ from djeuscan.feeds import PackageFeed, CategoryFeed, HerdFeed, \
MaintainerFeed, GlobalFeed MaintainerFeed, GlobalFeed
admin_required = user_passes_test(lambda u: u.is_superuser)
package_patterns = patterns('djeuscan.views', package_patterns = patterns('djeuscan.views',
url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/feed/$', url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/feed/$',
PackageFeed(), name='package_feed'), PackageFeed(), name='package_feed'),
url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/$', url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/$',
'package', name="package"), 'package', name="package"),
url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/favourite$',
'favourite_package', name="favourite_package"),
url((r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/'
'unfavourite$'),
'unfavourite_package', name="unfavourite_package"),
) )
categories_patterns = patterns('djeuscan.views', categories_patterns = patterns('djeuscan.views',
@ -16,6 +30,10 @@ categories_patterns = patterns('djeuscan.views',
name='category_feed'), name='category_feed'),
url(r'^(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', url(r'^(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$',
'chart_category', name="chart_category"), 'chart_category', name="chart_category"),
url(r'^(?P<category>[\w+][\w+.-]*)/favourite$',
'favourite_category', name="favourite_category"),
url(r'^(?P<category>[\w+][\w+.-]*)/unfavourite$',
'unfavourite_category', name="unfavourite_category"),
url(r'^$', 'categories', name="categories"), url(r'^$', 'categories', name="categories"),
) )
@ -24,6 +42,10 @@ herds_patterns = patterns('djeuscan.views',
url(r'^(?P<herd>[\@\{\}\w+.-]*)/feed/$', HerdFeed(), name='herd_feed'), url(r'^(?P<herd>[\@\{\}\w+.-]*)/feed/$', HerdFeed(), name='herd_feed'),
url(r'^(?P<herd>[\@\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', url(r'^(?P<herd>[\@\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$',
'chart_herd', name="chart_herd"), 'chart_herd', name="chart_herd"),
url(r'^(?P<herd>[\@\{\}\w+.-]*)/favourite$', 'favourite_herd',
name="favourite_herd"),
url(r'^(?P<herd>[\@\{\}\w+.-]*)/unfavourite$', 'unfavourite_herd',
name="unfavourite_herd"),
url(r'^$', 'herds', name="herds"), url(r'^$', 'herds', name="herds"),
) )
@ -33,6 +55,10 @@ maintainers_patterns = patterns('djeuscan.views',
name='maintainer_feed'), name='maintainer_feed'),
url(r'^(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$', url(r'^(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$',
'chart_maintainer', name="chart_maintainer"), 'chart_maintainer', name="chart_maintainer"),
url(r'^(?P<maintainer_id>\d+)/favourite$',
'favourite_maintainer', name="favourite_maintainer"),
url(r'^(?P<maintainer_id>\d+)/unfavourite$',
'unfavourite_maintainer', name="unfavourite_maintainer"),
url(r'^$', 'maintainers', name="maintainers"), url(r'^$', 'maintainers', name="maintainers"),
) )
@ -41,6 +67,26 @@ overlays_patterns = patterns('djeuscan.views',
url(r'^$', 'overlays', name="overlays"), url(r'^$', 'overlays', name="overlays"),
) )
tasks_patterns = patterns('djeuscan.views',
url(r'^refresh_package/(?P<query>(?:[\w+][\w+.-]*/[\w+][\w+.-]*))/$',
"refresh_package",
name="refresh_package"),
url(r'^registered_tasks/$', admin_required(registered_tasks),
name="registered_tasks"),
url(r'^apply/(?P<task_name>.*)/$', admin_required(apply_task),
name="apply_task"),
)
accounts_patterns = patterns('djeuscan.views',
url(r'^profile/$', 'accounts_index', name="accounts_index"),
url(r'^categories/$', 'accounts_categories', name="accounts_categories"),
url(r'^herds/$', 'accounts_herds', name="accounts_herds"),
url(r'^maintainers/$', 'accounts_maintainers',
name="accounts_maintainers"),
url(r'^packages/$', 'accounts_packages', name="accounts_packages"),
)
urlpatterns = patterns('djeuscan.views', urlpatterns = patterns('djeuscan.views',
# Global stuff # Global stuff
url(r'^api/', include('djeuscan.api.urls')), url(r'^api/', include('djeuscan.api.urls')),
@ -60,4 +106,7 @@ urlpatterns = patterns('djeuscan.views',
url(r'^maintainers/', include(maintainers_patterns)), url(r'^maintainers/', include(maintainers_patterns)),
url(r'^overlays/', include(overlays_patterns)), url(r'^overlays/', include(overlays_patterns)),
url(r'^package/', include(package_patterns)), url(r'^package/', include(package_patterns)),
url(r'^tasks/', include(tasks_patterns)),
url(r'^accounts/', include(accounts_patterns)),
) )

View File

@ -1,13 +1,19 @@
""" Views """ """ Views """
from annoying.decorators import render_to import inspect
from annoying.decorators import render_to, ajax_request
from django.http import Http404 from django.http import Http404
from django.shortcuts import get_object_or_404 from django.shortcuts import get_object_or_404
from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_POST
from djeuscan.helpers import version_key, packages_from_names from djeuscan.helpers import version_key, packages_from_names
from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \ from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
VersionLog VersionLog, RefreshPackageQuery, HerdAssociation, MaintainerAssociation, \
CategoryAssociation, PackageAssociation
from djeuscan.forms import WorldForm, PackagesForm from djeuscan.forms import WorldForm, PackagesForm
from djeuscan.tasks import admin_tasks
from djeuscan import charts from djeuscan import charts
@ -161,13 +167,23 @@ def package(request, category, package):
except EuscanResult.DoesNotExist: except EuscanResult.DoesNotExist:
last_scan = None last_scan = None
favourited = False
if request.user.is_authenticated():
try:
PackageAssociation.objects.get(user=request.user, package=package)
except PackageAssociation.DoesNotExist:
pass
else:
favourited = True
return { return {
'package': package, 'package': package,
'packaged': packaged, 'packaged': packaged,
'upstream': upstream, 'upstream': upstream,
'log': log, 'log': log,
'vlog': vlog, 'vlog': vlog,
'last_scan': last_scan 'last_scan': last_scan,
'favourited': favourited,
} }
@ -254,3 +270,164 @@ def chart_herd(request, **kwargs):
def chart_category(request, **kwargs): def chart_category(request, **kwargs):
return chart(request, **kwargs) return chart(request, **kwargs)
@ajax_request
def registered_tasks(request):
data = {}
for task in admin_tasks:
argspec = inspect.getargspec(task.run)
data[task.name] = {
"args": argspec.args,
"defaults": argspec.defaults,
"default_types": None
}
if argspec.defaults is not None:
data[task.name].update({
"defaults_types": [type(x).__name__ for x in argspec.defaults]
})
return {"tasks": data}
@login_required
@require_POST
@ajax_request
def refresh_package(request, query):
obj, created = RefreshPackageQuery.objects.get_or_create(query=query)
if not created:
obj.priority += 1
obj.save()
return {"result": "success"}
@login_required
@render_to('euscan/accounts/index.html')
def accounts_index(request):
return {}
@login_required
@render_to('euscan/accounts/categories.html')
def accounts_categories(request):
categories = [obj.category for obj in
CategoryAssociation.objects.filter(user=request.user)]
return {"categories": categories}
@login_required
@render_to('euscan/accounts/herds.html')
def accounts_herds(request):
herds = [obj.herd for obj in
HerdAssociation.objects.filter(user=request.user)]
return {"herds": herds}
@login_required
@render_to('euscan/accounts/maintainers.html')
def accounts_maintainers(request):
maintainers = [obj.maintainer for obj in
MaintainerAssociation.objects.filter(user=request.user)]
return {"maintainers": maintainers}
@login_required
@render_to('euscan/accounts/packages.html')
def accounts_packages(request):
packages = [obj.package for obj in
PackageAssociation.objects.filter(user=request.user)]
return {"packages": packages}
@login_required
@require_POST
@ajax_request
def favourite_package(request, category, package):
obj = get_object_or_404(Package, category=category, name=package)
_, created = PackageAssociation.objects.get_or_create(
user=request.user, package=obj
)
return {"success": created}
@login_required
@require_POST
@ajax_request
def unfavourite_package(request, category, package):
package = get_object_or_404(Package, category=category, name=package)
obj = get_object_or_404(
PackageAssociation, package=package, user=request.user
)
obj.delete()
return {"success": True}
@login_required
@require_POST
@ajax_request
def favourite_herd(request, herd):
obj = get_object_or_404(Herd, herd=herd)
_, created = HerdAssociation.objects.get_or_create(
user=request.user, herd=obj
)
return {"success": created}
@login_required
@require_POST
@ajax_request
def unfavourite_herd(request, herd):
herd = get_object_or_404(Herd, herd=herd)
obj = get_object_or_404(
HerdAssociation, herd=herd, user=request.user
)
obj.delete()
return {"success": True}
@login_required
@require_POST
@ajax_request
def favourite_maintainer(request, maintainer_id):
obj = get_object_or_404(Maintainer, pk=maintainer_id)
_, created = MaintainerAssociation.objects.get_or_create(
user=request.user, maintainer=obj
)
return {"success": created}
@login_required
@require_POST
@ajax_request
def unfavourite_maintainer(request, maintainer_id):
maintainer = get_object_or_404(Maintainer, pk=maintainer_id)
obj = get_object_or_404(
MaintainerAssociation, maintainer=maintainer, user=request.user
)
obj.delete()
return {"success": True}
@login_required
@require_POST
@ajax_request
def favourite_category(request, category):
packages = Package.objects.for_category(category, last_versions=True)
if not packages:
raise Http404
_, created = CategoryAssociation.objects.get_or_create(
user=request.user, category=category
)
return {"success": created}
@login_required
@require_POST
@ajax_request
def unfavourite_category(request, category):
obj = get_object_or_404(
CategoryAssociation, user=request.user, category=category
)
obj.delete()
return {"success": True}

View File

@ -157,6 +157,9 @@ TEMPLATE_CONTEXT_PROCESSORS = (
) )
INSTALLED_APPS = ( INSTALLED_APPS = (
'euscanwww',
'djeuscan',
'django.contrib.auth', 'django.contrib.auth',
'django.contrib.contenttypes', 'django.contrib.contenttypes',
'django.contrib.sessions', 'django.contrib.sessions',
@ -166,8 +169,8 @@ INSTALLED_APPS = (
# Uncomment the next line to enable admin documentation: # Uncomment the next line to enable admin documentation:
# 'django.contrib.admindocs', # 'django.contrib.admindocs',
'south', 'south',
'euscanwww', 'djcelery',
'djeuscan', 'registration',
) )
# A sample logging configuration. The only tangible logging # A sample logging configuration. The only tangible logging
@ -178,12 +181,22 @@ INSTALLED_APPS = (
LOGGING = { LOGGING = {
'version': 1, 'version': 1,
'disable_existing_loggers': False, 'disable_existing_loggers': False,
'formatters': {
'simple': {
'format': '%(levelname)s %(asctime)s %(message)s'
},
},
'filters': { 'filters': {
'require_debug_false': { 'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse' '()': 'django.utils.log.RequireDebugFalse'
} }
}, },
'handlers': { 'handlers': {
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'simple'
},
'mail_admins': { 'mail_admins': {
'level': 'ERROR', 'level': 'ERROR',
'filters': ['require_debug_false'], 'filters': ['require_debug_false'],
@ -196,14 +209,49 @@ LOGGING = {
'level': 'ERROR', 'level': 'ERROR',
'propagate': True, 'propagate': True,
}, },
'djeuscan': {
'handlers': ['console'],
'level': 'INFO',
'propagate': True
}
} }
} }
# django-registration
ACCOUNT_ACTIVATION_DAYS = 7
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
# djeuscan tasks
PORTAGE_ROOT = "/usr/portage/"
PORTAGE_CONFIGROOT = PORTAGE_ROOT
LAYMAN_CONFIG = "/etc/layman/layman.cfg"
EMERGE_REGEN_JOBS = 4
# Celery config
import djcelery
djcelery.setup_loader()
BROKER_URL = "amqp://guest:guest@localhost:5672//"
CELERY_RESULT_BACKEND = "amqp"
BROKER_CONNECTION_TIMEOUT = 3600
CELERYD_CONCURRENCY = 4
TASKS_CONCURRENTLY = 4
TASKS_SUBTASK_PACKAGES = 32
# LDAP authentication
# TODO: Test data - change me!
AUTH_LDAP_SERVER_URI = "ldap://localhost"
AUTH_LDAP_USER_DN_TEMPLATE = "uid=%(user)s,ou=users,dc=my-domain,dc=com"
AUTHENTICATION_BACKENDS = (
'django_auth_ldap.backend.LDAPBackend',
'django.contrib.auth.backends.ModelBackend',
)
try: try:
from local_settings import * from local_settings import *
except ImportError, ex: except ImportError, ex:
import sys import sys
sys.stderr.write(\ sys.stderr.write(
("settings.py: error importing local settings file:\n" + \ "settings.py: error importing local settings file:\n"
"\t%s\n" + \ "\t%s\nDo you have a local_settings.py module?\n" % str(ex)
"Do you have a local_settings.py module?\n") % str(ex)) )

View File

@ -10,6 +10,7 @@ urlpatterns = patterns('',
# (r'^admin/doc/', include('django.contrib.admindocs.urls')), # (r'^admin/doc/', include('django.contrib.admindocs.urls')),
url(r'^admin/', include(admin.site.urls)), url(r'^admin/', include(admin.site.urls)),
url(r'^accounts/', include('registration.backends.default.urls')),
url(r'^', include('djeuscan.urls')), url(r'^', include('djeuscan.urls')),
) )

File diff suppressed because one or more lines are too long

View File

@ -1,96 +1,80 @@
body body {
{
margin: 0;
padding: 0;
font-size: 0.8em; font-size: 0.8em;
font-family: Dejavu, Verdana, "Bitstream Vera Sans", "Lucida Grande", "Trebuchet MS", sans-serif; font-family: Dejavu, Verdana, "Bitstream Vera Sans", "Lucida Grande", "Trebuchet MS", sans-serif;
color: #535353; color: #535353;
background: #D2D0D4; background: #D2D0D4;
}
img
{
border: none;
}
h1
{
margin-top: 0;
color: #369;
font-size: 1.6em;
} }
a:link, a:visited img {
{ border: none;
}
h1 {
margin-top: 0;
color: #333;
font-size: 1.6em;
width: 20em;
margin-left: auto;
margin-right: auto;
}
a:link, a:visited {
text-decoration: none; text-decoration: none;
font-weight: bold; font-weight: bold;
color: #ff8c00; color: #ff8c00;
} }
a:hover, a:active a:hover, a:active {
{
font-weight: bold; font-weight: bold;
color: #ff4500; color: #ff4500;
text-decoration: underline; text-decoration: underline;
} }
#header { #header {
/* background: url(http://packages.gentoo.org/media/packages_gentoo_logo.jpg) no-repeat;
background-color: #46347C; */
width: 100%; width: 100%;
height: 30px; height: 30px;
margin: 0; margin: 0;
padding: 0; padding: 0;
background: url(../img/gentoo-header-bar-bg.png) repeat-x; background: url(../img/gentoo-header-bar-bg.png) repeat-x;
margin-bottom: 20px;
} }
#header #logo #header #logo {
{
float: left; float: left;
} }
#content { #content {
background: #F0F0F0;
padding: 10px;
margin-top: 10px;
/* margin-right: auto;
margin-left: auto;
*/
margin-left: 20px;
/* margin-left: 20%; */
margin-right: 250px;
/* max-width: 60em; */
border: 1px solid #67539B; border: 1px solid #67539B;
background: #FEFEFE; background: #FEFEFE;
-webkit-border-radius: 10px; padding: 20px;
-moz-border-radius: 10px;
border-radius: 10px; min-height: 350px;
} }
.menu { .menu {
-webkit-border-radius: 10px;
-moz-border-radius: 10px;
border-radius: 10px;
border: 1px solid #67539B; border: 1px solid #67539B;
}
.menu {
width:100%;
margin-top: 30px;
background: #8076A1; background: #8076A1;
color: #FFF; color: #FFF;
padding-top: 5px;
padding-bottom: 5px;
}
.notfirst-menu {
margin-top: 10px;
} }
.menu a { .menu a {
color: #fff; color: #fff;
} }
.menu li .menu li {
{
list-style-type: none; list-style-type: none;
} }
.submenu {
padding-left: 15px;
}
.menu dl { .menu dl {
margin-left: 15px; margin-left: 15px;
@ -100,29 +84,7 @@ a:hover, a:active
border: none; border: none;
} }
#menus { a {
width: 200px;
position:absolute;
top:30px;
right:20px;
/*
position: fixed;
right: 10px;
top: 2em;
width: 20%;
*/
}
/*
code, pre{
background-color:transparent;
font-family:"Courier New",Courier,monospace;
font-size:small;
}
*/
a{
color: #3F4C66; color: #3F4C66;
} }
@ -139,7 +101,7 @@ abbr:hover {
text-decoration: underline; text-decoration: underline;
} }
pre{ pre {
border-left:5px solid; border-left:5px solid;
padding:0.5em 1em; padding:0.5em 1em;
margin-left:2em; margin-left:2em;
@ -155,29 +117,15 @@ dd {
padding: 5px; padding: 5px;
} }
hr {
h1 {
color: #000;
width: 20em;
margin-left: auto;
margin-right: auto;
}
.ok {
color:#15B100;
}
hr
{
margin: 0.3em 1em 0.3em 1em; margin: 0.3em 1em 0.3em 1em;
height: 1px; height: 1px;
border: #bcbcbc dashed; border: #bcbcbc dashed;
border-width: 0 0 1px 0; border-width: 0 0 1px 0;
} }
table table {
{
max-width: 60em; max-width: 60em;
/* width: 50%; */ /* width: 50%; */
border-collapse: collapse; border-collapse: collapse;
@ -188,42 +136,38 @@ table
text-align: left; text-align: left;
} }
td td {
{
vertical-align: top; vertical-align: top;
} }
th th {
{
font-size: 0.8em; font-size: 0.8em;
text-align: center; text-align: center;
border-bottom: 3px solid; border-bottom: 3px solid;
} }
#footer #footer {
{
background: #8076A1; background: #8076A1;
font-size: 0.8em; font-size: 0.8em;
width: 100%; width: 100%;
margin: 0; margin: 0;
margin-top: 10px; margin-top: 20px;
color: #FFF; color: #FFF;
} }
#footer p { #footer p {
text-align: right; text-align: right;
margin: 5px; margin: 5px;
} }
.err,.ok,.inf .err, .ok, .inf {
{
margin: 5px; margin: 5px;
margin-left: auto; margin-left: auto;
margin-right: auto; margin-right: auto;
width: 70%; width: 70%;
font-weight:bold; font-weight:bold;
border: 1px dotted #5682AD; border: 1px dotted #5682AD;
} }
.added { .added {
color: #262; color: #262;
@ -233,17 +177,15 @@ th
color: #F00; color: #F00;
} }
.err .err {
{
border-color: #F00; border-color: #F00;
color: #F00; color: #F00;
} }
.ok .ok {
{
border-color: #262; border-color: #262;
color: #262; color: #262;
} }
.logo { .logo {
float: right; float: right;
@ -281,3 +223,13 @@ th
max-height: 100pt; max-height: 100pt;
overflow: auto; overflow: auto;
} }
.hide {
display: none;
}
.rounded {
-webkit-border-radius: 10px;
-moz-border-radius: 10px;
border-radius: 10px;
}

View File

@ -1,6 +1,7 @@
#!/usr/bin/python #!/usr/bin/python
import sys, os import sys
import os
import os.path import os.path
PROJECT = '/path/to/euscanwww' PROJECT = '/path/to/euscanwww'

View File

@ -19,7 +19,8 @@ CONFIG = {
'skip-robots-txt': False, 'skip-robots-txt': False,
'cache': False, 'cache': False,
'format': None, 'format': None,
'indent': 2 'indent': 2,
'progress': False
} }
BLACKLIST_VERSIONS = [ BLACKLIST_VERSIONS = [
@ -67,4 +68,4 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [
] ]
from out import EuscanOutput from out import EuscanOutput
output = out.EuscanOutput(CONFIG) output = EuscanOutput(CONFIG)

View File

@ -1,6 +1,17 @@
from euscan.handlers import generic, php, pypi, rubygem, kde, cpan, github import pkgutil
handlers = [kde, php, pypi, rubygem, cpan, github, generic] # autoimport all modules in this directory and append them to handlers list
handlers = []
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
module = loader.find_module(module_name).load_module(module_name)
handlers.append(module)
# sort handlers by priority (e.g.: generic should be run lastly)
handlers = sorted(
handlers,
key=lambda handler: handler.PRIORITY,
reverse=True
)
def find_best_handler(cpv, url): def find_best_handler(cpv, url):

View File

@ -7,6 +7,7 @@ from euscan import helpers, output
HANDLER_NAME = "cpan" HANDLER_NAME = "cpan"
CONFIDENCE = 100.0 CONFIDENCE = 100.0
PRIORITY = 100
_cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*") _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*")

View File

@ -14,6 +14,7 @@ from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \
HANDLER_NAME = "generic" HANDLER_NAME = "generic"
CONFIDENCE = 50.0 CONFIDENCE = 50.0
PRIORITY = 0
BRUTEFORCE_HANDLER_NAME = "brute_force" BRUTEFORCE_HANDLER_NAME = "brute_force"
BRUTEFORCE_CONFIDENCE = 30.0 BRUTEFORCE_CONFIDENCE = 30.0

View File

@ -1,4 +1,5 @@
import json, urllib2 import json
import urllib2
import re import re
import portage import portage
@ -7,16 +8,20 @@ from euscan import helpers, output
HANDLER_NAME = "github" HANDLER_NAME = "github"
CONFIDENCE = 100.0 CONFIDENCE = 100.0
PRIORITY = 100
def can_handle(cpv, url): def can_handle(cpv, url):
return url.startswith('mirror://github/') return url.startswith('mirror://github/')
def guess_package(cp, url): def guess_package(cp, url):
match = re.search('^mirror://github/(.*?)/(.*?)/(.*)$', url) match = re.search('^mirror://github/(.*?)/(.*?)/(.*)$', url)
assert(match) assert(match)
return (match.group(1), match.group(2), match.group(3)) return (match.group(1), match.group(2), match.group(3))
def scan(cpv, url): def scan(cpv, url):
'http://developer.github.com/v3/repos/downloads/' 'http://developer.github.com/v3/repos/downloads/'
@ -30,11 +35,13 @@ def scan(cpv, url):
# now create a filename-matching regexp # now create a filename-matching regexp
# XXX: supposedly replace first with (?P<foo>...) # XXX: supposedly replace first with (?P<foo>...)
# and remaining ones with (?P=foo) # and remaining ones with (?P=foo)
fnre = re.compile('^%s$' % re.escape(filename).replace(re.escape(ver), '(.*?)')) fnre = re.compile('^%s$' % \
re.escape(filename).replace(re.escape(ver), '(.*?)'))
output.einfo("Using github API for: " + '/'.join(package)) output.einfo("Using github API for: " + '/'.join(filename))
dlreq = urllib2.urlopen('https://api.github.com/repos/%s/%s/downloads' % (user, project)) dlreq = urllib2.urlopen('https://api.github.com/repos/%s/%s/downloads' % \
(user, project))
dls = json.load(dlreq) dls = json.load(dlreq)
for dl in dls: for dl in dls:
@ -46,5 +53,6 @@ def scan(cpv, url):
continue continue
yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE) yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE)
def brute_force(cpv, url): def brute_force(cpv, url):
return [] return []

View File

@ -1,5 +1,7 @@
from euscan.handlers import generic from euscan.handlers import generic
PRIORITY = 100
HANDLER_NAME = "kde" HANDLER_NAME = "kde"

View File

@ -7,6 +7,7 @@ from euscan import helpers, output
HANDLER_NAME = "php" HANDLER_NAME = "php"
CONFIDENCE = 100.0 CONFIDENCE = 100.0
PRIORITY = 100
def can_handle(cpv, url): def can_handle(cpv, url):

View File

@ -7,6 +7,7 @@ from euscan import helpers, output
HANDLER_NAME = "pypi" HANDLER_NAME = "pypi"
CONFIDENCE = 100.0 CONFIDENCE = 100.0
PRIORITY = 100
def can_handle(cpv, url): def can_handle(cpv, url):

View File

@ -7,6 +7,7 @@ from euscan import helpers, output
HANDLER_NAME = "rubygem" HANDLER_NAME = "rubygem"
CONFIDENCE = 100.0 CONFIDENCE = 100.0
PRIORITY = 100
def can_handle(cpv, url): def can_handle(cpv, url):

View File

@ -44,7 +44,7 @@ def get_version_type(version):
if token in gentoo_types: if token in gentoo_types:
types.append(token) types.append(token)
if types: if types:
return types[0] return types[0] # TODO: consider returning all types
return "release" return "release"

View File

@ -3,6 +3,7 @@ from collections import defaultdict
import json import json
import signal import signal
import time import time
import re
from gentoolkit import pprinter as pp from gentoolkit import pprinter as pp
import portage import portage
@ -10,15 +11,20 @@ from portage.output import EOutput, TermProgressBar
class ProgressHandler(object): class ProgressHandler(object):
def __init__(self): def __init__(self, progress_bar):
self.curval = 0 self.curval = 0
self.maxval = 0 self.maxval = 0
self.last_update = 0 self.last_update = 0
self.min_display_latency = 0.2 self.min_display_latency = 0.2
self.progress_bar = progress_bar
def on_progress(self, maxval=None, increment=1, label=None):
self.maxval = maxval or self.maxval
self.curval += increment
if label:
self.progress_bar.label(label)
def on_progress(self, maxval, curval):
self.maxval = maxval
self.curval = curval
cur_time = time.time() cur_time = time.time()
if cur_time - self.last_update >= self.min_display_latency: if cur_time - self.last_update >= self.min_display_latency:
self.last_update = cur_time self.last_update = cur_time
@ -32,7 +38,7 @@ def progress_bar():
on_progress = None on_progress = None
progress_bar = TermProgressBar() progress_bar = TermProgressBar()
progress_handler = ProgressHandler() progress_handler = ProgressHandler(progress_bar)
on_progress = progress_handler.on_progress on_progress = progress_handler.on_progress
def display(): def display():
@ -61,12 +67,20 @@ class EOutputMem(EOutput):
self.out = StringIO() self.out = StringIO()
def getvalue(self): def getvalue(self):
return self.out.getvalue() return clean_colors(self.out.getvalue())
def _write(self, f, msg): def _write(self, f, msg):
super(EOutputMem, self)._write(self.out, msg) super(EOutputMem, self)._write(self.out, msg)
def clean_colors(string):
if type(string) is str:
string = re.sub("\033\[[0-9;]+m", "", string)
string = re.sub(r"\\u001b\[[0-9;]+m", "", string)
string = re.sub(r"\x1b\[[0-9;]+m", "", string)
return string
class EuscanOutput(object): class EuscanOutput(object):
""" """
Class that handles output for euscan Class that handles output for euscan
@ -76,6 +90,10 @@ class EuscanOutput(object):
self.queries = defaultdict(dict) self.queries = defaultdict(dict)
self.current_query = None self.current_query = None
def clean(self):
self.queries = defaultdict(dict)
self.current_query = None
def set_query(self, query): def set_query(self, query):
self.current_query = query self.current_query = query
if query is None: if query is None:
@ -95,7 +113,7 @@ class EuscanOutput(object):
"metadata": {}, "metadata": {},
} }
def get_formatted_output(self): def get_formatted_output(self, format_=None):
data = {} data = {}
for query in self.queries: for query in self.queries:
@ -105,8 +123,11 @@ class EuscanOutput(object):
"messages": self.queries[query]["output"].getvalue(), "messages": self.queries[query]["output"].getvalue(),
} }
if self.config["format"].lower() == "json": format_ = format_ or self.config["format"]
if format_.lower() == "json":
return json.dumps(data, indent=self.config["indent"]) return json.dumps(data, indent=self.config["indent"])
elif format_.lower() == "dict":
return data
else: else:
raise TypeError("Invalid output format") raise TypeError("Invalid output format")
@ -128,12 +149,13 @@ class EuscanOutput(object):
def metadata(self, key, value, show=True): def metadata(self, key, value, show=True):
if self.config["format"]: if self.config["format"]:
self.queries[self.current_query]["metadata"][key] = value self.queries[self.current_query]["metadata"][key] = \
clean_colors(value)
elif show: elif show:
print "%s: %s" % (key.capitalize(), value) print "%s: %s" % (key.capitalize(), value)
def __getattr__(self, key): def __getattr__(self, key):
if not self.config["quiet"]: if not self.config["quiet"] and self.current_query is not None:
output = self.queries[self.current_query]["output"] output = self.queries[self.current_query]["output"]
return getattr(output, key) return getattr(output, key)
else: else:

View File

@ -1,4 +1,7 @@
from __future__ import print_function
import os import os
import sys
from datetime import datetime from datetime import datetime
import portage import portage
@ -42,15 +45,21 @@ def filter_versions(cp, versions):
def scan_upstream_urls(cpv, urls, on_progress): def scan_upstream_urls(cpv, urls, on_progress):
versions = [] versions = []
maxval = len(urls) + 5 if on_progress:
curval = 1 progress_available = 70
num_urls = sum([len(urls[fn]) for fn in urls])
if num_urls > 0:
progress_increment = progress_available / num_urls
else:
progress_increment = 0
for filename in urls: for filename in urls:
curval += 1
if on_progress:
on_progress(maxval, curval)
for url in urls[filename]: for url in urls[filename]:
if on_progress and progress_available > 0:
on_progress(increment=progress_increment)
progress_available -= progress_increment
if not CONFIG['quiet'] and not CONFIG['format']: if not CONFIG['quiet'] and not CONFIG['format']:
pp.uprint() pp.uprint()
output.einfo("SRC_URI is '%s'" % url) output.einfo("SRC_URI is '%s'" % url)
@ -79,15 +88,10 @@ def scan_upstream_urls(cpv, urls, on_progress):
cp, ver, rev = portage.pkgsplit(cpv) cp, ver, rev = portage.pkgsplit(cpv)
curval += 1
if on_progress:
on_progress(maxval, curval)
result = filter_versions(cp, versions) result = filter_versions(cp, versions)
curval += 1 if on_progress and progress_available > 0:
if on_progress: on_progress(increment=progress_available)
on_progress(maxval, curval)
return result return result
@ -115,9 +119,6 @@ def scan_upstream(query, on_progress=None):
Scans the upstream searching new versions for the given query Scans the upstream searching new versions for the given query
""" """
maxval = 3
curval = 0
matches = [] matches = []
if query.endswith(".ebuild"): if query.endswith(".ebuild"):
@ -156,9 +157,8 @@ def scan_upstream(query, on_progress=None):
output.metadata("cp", pkg.cp, show=False) output.metadata("cp", pkg.cp, show=False)
output.metadata("cpv", pkg.cpv, show=False) output.metadata("cpv", pkg.cpv, show=False)
curval += 1
if on_progress: if on_progress:
on_progress(maxval, curval) on_progress(increment=10)
if pkg.cp in BLACKLIST_PACKAGES: if pkg.cp in BLACKLIST_PACKAGES:
output.ewarn( output.ewarn(
@ -210,14 +210,16 @@ def scan_upstream(query, on_progress=None):
scan_time = (datetime.now() - start_time).total_seconds() scan_time = (datetime.now() - start_time).total_seconds()
output.metadata("scan_time", scan_time, show=False) output.metadata("scan_time", scan_time, show=False)
curval += 1
if on_progress:
on_progress(maxval, curval)
result = scan_upstream_urls(pkg.cpv, urls, on_progress) result = scan_upstream_urls(pkg.cpv, urls, on_progress)
curval += 1
if on_progress: if on_progress:
on_progress(maxval, curval) on_progress(increment=10)
if len(result) > 0:
if not (CONFIG['format'] or CONFIG['quiet']):
print("\n", file=sys.stderr)
for cp, url, version, handler, confidence in result:
output.result(cp, version, url, handler, confidence)
return result return result

View File

@ -87,9 +87,12 @@ setup(
'https://github.com/iksaif/euscan/tarball/' + 'https://github.com/iksaif/euscan/tarball/' +
('master' if __version__ == '9999' else ('euscan-%s' % __version__)) ('master' if __version__ == '9999' else ('euscan-%s' % __version__))
), ),
install_requires=['Django==1.4', 'django-annoying==0..7.6', 'South==0.7.4', install_requires=[
'django-piston==0.2.3', 'BeautifulSoup==3.2.1', 'Django==1.4', 'django-annoying==0.7.6', 'South==0.7.4',
'matplotlib==1.1.0', 'django-celery==2.5.5'], 'django-piston==0.2.3', 'BeautifulSoup==3.2.1', 'matplotlib==1.1.0',
'django-celery==2.5.5', 'django-registration==0.8',
'python-ldap==2.4.10', 'django-auth-ldap==1.1',
],
package_dir={'': 'pym'}, package_dir={'': 'pym'},
packages=packages, packages=packages,
package_data={}, package_data={},