src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #2 from volpino/master

Browse Source 
Cleaning up and updating requirements
This commit is contained in:
Corentin Chary 2012-04-29 12:42:14 -07:00
commit 1a40cf7bcc
46 changed files with 796 additions and 451 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README
View File

@ -88,6 +88,15 @@ tree to avoid messing with your system.
### Installation ### Installation
Install requirements from PyPI using
$ python setup.py develop
Extra dependencies:
* portage python api
* rrdtool[python]
Like any django web app, just start by editing settings.py and then run Like any django web app, just start by editing settings.py and then run
these two commands. these two commands.

96
bin/euscan
View File

@ -34,11 +34,12 @@ from euscan.scan import scan_upstream
""" Globals """ """ Globals """
def setupSignals(): def setupSignals():
""" This block ensures that ^C interrupts are handled quietly. """ """ This block ensures that ^C interrupts are handled quietly. """
import signal import signal
def exithandler(signum,frame): def exithandler(signum, frame):
signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN)
print () print ()
@ -51,10 +52,10 @@ def setupSignals():
def printVersion(): def printVersion():
"""Output the version info.""" """Output the version info."""
print( "%s (%s) - %s" \ print("%s (%s) - %s" \
% (__productname__, __version__, __description__)) % (__productname__, __version__, __description__))
print() print()
print("Author: %s <%s>" % (__author__,__email__)) print("Author: %s <%s>" % (__author__, __email__))
print("Copyright 2011 Gentoo Foundation") print("Copyright 2011 Gentoo Foundation")
print("Distributed under the terms of the GNU General Public License v2") print("Distributed under the terms of the GNU General Public License v2")
@ -68,45 +69,49 @@ def printUsage(_error=None, help=None):
out = sys.stderr out = sys.stderr
if not _error in ('global-options', 'packages',): if not _error in ('global-options', 'packages',):
_error = None _error = None
if not _error and not help: help = 'all' if not _error and not help:
help = 'all'
if _error in ('global-options',): if _error in ('global-options',):
print( pp.error("Wrong option on command line."), file=out) print(pp.error("Wrong option on command line."), file=out)
print( file=out) print(file=out)
if _error in ('packages',): if _error in ('packages',):
print( pp.error("You need to specify exactly one package."), file=out) print(pp.error("You need to specify exactly one package."), file=out)
print( file=out) print(file=out)
print( white("Usage:"), file=out) print(white("Usage:"), file=out)
if _error in ('global-options', 'packages',) or help == 'all': if _error in ('global-options', 'packages',) or help == 'all':
print( " "+turquoise(__productname__), print(" " + turquoise(__productname__),
yellow("[options]"), yellow("[options]"),
green("<package> [<package> [...]]"), file=out) green("<package> [<package> [...]]"), file=out)
if _error in ('global-options',) or help == 'all': if _error in ('global-options',) or help == 'all':
print( " "+turquoise(__productname__), print(" " + turquoise(__productname__),
yellow("[--help, --version]"), file=out) yellow("[--help, --version]"), file=out)
print(file=out) print(file=out)
if _error in ('global-options',) or help: if _error in ('global-options',) or help:
print( "Available ", yellow("options")+":", file=out) print("Available ", yellow("options") + ":", file=out)
print( yellow(" -C, --nocolor")+ print(yellow(" -C, --nocolor") +
" - turn off colors on output", file=out) " - turn off colors on output", file=out)
print( yellow(" -q, --quiet")+ print(yellow(" -q, --quiet") +
" - be as quiet as possible", file=out) " - be as quiet as possible", file=out)
print( yellow(" -h, --help")+ \ print(yellow(" -h, --help") +
" - display the help screen", file=out) " - display the help screen", file=out)
print( yellow(" -V, --version")+ print(yellow(" -V, --version") +
" - display version info", file=out) " - display version info", file=out)
print( file=out) print(file=out)
print( yellow(" -1, --oneshot")+ print(yellow(" -1, --oneshot") +
" - stop as soon as a new version is found", file=out) " - stop as soon as a new version is found",
print( yellow(" -b, --brute-force=<level>")+ file=out)
" - define the brute force "+yellow("<level>")+" (default: 2)\n" + print(yellow(" -b, --brute-force=<level>") +
" " * 29 + "bigger levels will generate more versions numbers\n" + " - define the brute force " + yellow("<level>") +
" " * 29 + "0 means disabled", file=out) " (default: 2)\n" +
print( file=out) " " * 29 + "bigger levels will generate more versions numbers\n" +
" " * 29 + "0 means disabled", file=out)
print(file=out)
if _error in ('packages',) or help: if _error in ('packages',) or help:
print( green(" package")+ print(green(" package") +
" - the packages (or ebuilds) you want to scan", file=out) " - the packages (or ebuilds) you want to scan",
print( file=out) file=out)
print(file=out)
'''print( "More detailed instruction can be found in", '''print( "More detailed instruction can be found in",
turquoise("`man %s`" % __productname__), file=out)''' turquoise("`man %s`" % __productname__), file=out)'''
@ -115,6 +120,7 @@ class ParseArgsException(Exception):
"""For parseArgs() -> main() communications.""" """For parseArgs() -> main() communications."""
def __init__(self, value): def __init__(self, value):
self.value = value self.value = value
def __str__(self): def __str__(self):
return repr(self.value) return repr(self.value)
@ -130,7 +136,7 @@ def parseArgs():
return_code = True return_code = True
for o, a in opts: for o, a in opts:
if o in ("-h", "--help"): if o in ("-h", "--help"):
raise ParseArgsException('help') raise ParseArgsException('help')
elif o in ("-V", "--version"): elif o in ("-V", "--version"):
raise ParseArgsException('version') raise ParseArgsException('version')
elif o in ("-C", "--nocolor"): elif o in ("-C", "--nocolor"):
@ -151,7 +157,7 @@ def parseArgs():
return return_code return return_code
' here are the different allowed command line options (getopt args) ' ' here are the different allowed command line options (getopt args) '
getopt_options = {'short':{}, 'long':{}} getopt_options = {'short': {}, 'long': {}}
getopt_options['short']['global'] = "hVCqv1b:" getopt_options['short']['global'] = "hVCqv1b:"
getopt_options['long']['global'] = ["help", "version", "nocolor", "quiet", getopt_options['long']['global'] = ["help", "version", "nocolor", "quiet",
"verbose", "oneshot", "brute-force="] "verbose", "oneshot", "brute-force="]
@ -164,19 +170,20 @@ def parseArgs():
try: try:
opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts) opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts)
except: except:
raise ParseArgsException(opts_mode+'-options') raise ParseArgsException(opts_mode + '-options')
' set options accordingly ' ' set options accordingly '
optionSwitch(opts) optionSwitch(opts)
if len(args) < 1: if len(args) < 1:
raise ParseArgsException('packages') raise ParseArgsException('packages')
return args return args
def main(): def main():
"""Parse command line and execute all actions.""" """Parse command line and execute all actions."""
CONFIG['nocolor'] = (port_settings["NOCOLOR"] in ('yes','true') CONFIG['nocolor'] = (port_settings["NOCOLOR"] in ('yes', 'true')
or not sys.stdout.isatty()) or not sys.stdout.isatty())
if CONFIG['nocolor']: if CONFIG['nocolor']:
pp.output.nocolor() pp.output.nocolor()
@ -212,11 +219,16 @@ def main():
for candidate in pkgs: for candidate in pkgs:
print(candidate) print(candidate)
from os.path import basename # To get the short name from os.path import basename # To get the short name
print(file=sys.stderr) print(file=sys.stderr)
print(pp.error("The short ebuild name '%s' is ambiguous. Please specify" % basename(pkgs[0])), print(
file=sys.stderr, end="") pp.error(
"The short ebuild name '%s' is ambiguous. Please specify" \
% basename(pkgs[0])
),
file=sys.stderr, end=""
)
pp.die(1, "one of the above fully-qualified ebuild names instead.") pp.die(1, "one of the above fully-qualified ebuild names instead.")
except GentoolkitException as err: except GentoolkitException as err:
pp.die(1, '%s: %s' % (package, str(err))) pp.die(1, '%s: %s' % (package, str(err)))
@ -233,12 +245,12 @@ def main():
+ pp.path(" %s" % url)) + pp.path(" %s" % url))
else: else:
print (pp.cpv("%s-%s" % (cp, version)) print (pp.cpv("%s-%s" % (cp, version))
+ ": " + pp.path(url)) + ": " + pp.path(url))
if not len(ret) and not CONFIG['quiet']: if not len(ret) and not CONFIG['quiet']:
print (pp.warn("Didn't find any new version, " print (pp.warn("Didn't find any new version, "
+ "check package's homepage for " + "check package's homepage for "
+ "more informations")); + "more informations"))
if __name__ == "__main__": if __name__ == "__main__":
@ -246,6 +258,6 @@ if __name__ == "__main__":
setupSignals() setupSignals()
main() main()
except KeyboardInterrupt: except KeyboardInterrupt:
print( "Aborted.") print("Aborted.")
sys.exit(errno.EINTR) sys.exit(errno.EINTR)
sys.exit(0) sys.exit(0)

5
euscanwww/djeuscan/admin.py
View File

@ -1,11 +1,14 @@
from models import * from models import Package, Version, VersionLog, EuscanResult, Log, WorldLog, \
CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer
from django.contrib import admin from django.contrib import admin
admin.site.register(Herd) admin.site.register(Herd)
admin.site.register(Maintainer) admin.site.register(Maintainer)
class PackageAdmin(admin.ModelAdmin): class PackageAdmin(admin.ModelAdmin):
search_fields = ('category', 'name') search_fields = ('category', 'name')
admin.site.register(Package, PackageAdmin) admin.site.register(Package, PackageAdmin)
admin.site.register(Version) admin.site.register(Version)

17
euscanwww/djeuscan/api/emitters.py
View File

@ -1,15 +1,16 @@
from piston.emitters import Emitter, XMLEmitter from piston.emitters import XMLEmitter
class EuscanXMLEmitter(XMLEmitter): class EuscanXMLEmitter(XMLEmitter):
_parent = [] _parent = []
_known_parents = { _known_parents = {
'vlog' : 'version', 'vlog': 'version',
'herds' : 'herd', 'herds': 'herd',
'maintainers' : 'maintainer', 'maintainers': 'maintainer',
'packaged' : 'version', 'packaged': 'version',
'upstream' : 'version', 'upstream': 'version',
'packages' : 'package', 'packages': 'package',
'categories' : 'category' 'categories': 'category'
} }
def _push_parent(self, parent): def _push_parent(self, parent):

87
euscanwww/djeuscan/api/handlers.py
View File

@ -1,24 +1,26 @@
from piston.handler import AnonymousBaseHandler, BaseHandler from piston.handler import AnonymousBaseHandler
from piston.utils import rc, HttpStatusCode from piston.utils import rc
from django.db.models import Sum, Max from django.db.models import Sum, Max
from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ObjectDoesNotExist
from django.forms.models import model_to_dict from django.forms.models import model_to_dict
from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, VersionLog from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
from djeuscan.forms import WorldForm, PackagesForm VersionLog
# replace default XMLEmitter with ours # replace default XMLEmitter with ours
from piston.emitters import Emitter from piston.emitters import Emitter
from emitters import EuscanXMLEmitter from emitters import EuscanXMLEmitter
Emitter.register('xml', EuscanXMLEmitter, 'text/xml; charset=utf-8') Emitter.register('xml', EuscanXMLEmitter, 'text/xml; charset=utf-8')
def xint(i): def xint(i):
try: try:
return int(i) return int(i)
except: except:
return 0 return 0
def renameFields(vqs, fields): def renameFields(vqs, fields):
ret = [] ret = []
for n in vqs: for n in vqs:
@ -29,6 +31,7 @@ def renameFields(vqs, fields):
ret.append(n) ret.append(n)
return ret return ret
class catch_and_return(object): class catch_and_return(object):
def __init__(self, err, response): def __init__(self, err, response):
self.err = err self.err = err
@ -42,6 +45,7 @@ class catch_and_return(object):
return self.response return self.response
return wrapper return wrapper
# /api/1.0/ # /api/1.0/
class RootHandler(AnonymousBaseHandler): class RootHandler(AnonymousBaseHandler):
allowed_methods = ('GET',) allowed_methods = ('GET',)
@ -49,52 +53,70 @@ class RootHandler(AnonymousBaseHandler):
def read(self, request): def read(self, request):
return {'api-version': '1.0'} return {'api-version': '1.0'}
# /api/1.0/statistics # /api/1.0/statistics
class StatisticsHandler(AnonymousBaseHandler): class StatisticsHandler(AnonymousBaseHandler):
allowed_methods = ('GET',) allowed_methods = ('GET',)
def read(self, request): def read(self, request):
_aggr = Package.objects.aggregate
data = {} data = {}
data['n_packaged'] = xint(Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum']) data['n_packaged'] = xint(_aggr(Sum('n_packaged'))['n_packaged__sum'])
data['n_overlay'] = xint(Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum']) data['n_overlay'] = xint(_aggr(Sum('n_overlay'))['n_overlay__sum'])
data['n_versions'] = xint(Package.objects.aggregate(Sum('n_versions'))['n_versions__sum']) data['n_versions'] = xint(_aggr(Sum('n_versions'))['n_versions__sum'])
data['n_upstream'] = data['n_versions'] - data['n_packaged'] - data['n_overlay'] data['n_upstream'] = data['n_versions'] - data['n_packaged'] - \
data['n_overlay']
data['n_packages'] = Package.objects.count() data['n_packages'] = Package.objects.count()
data['n_herds'] = Herd.objects.count() data['n_herds'] = Herd.objects.count()
data['n_maintainers'] = Maintainer.objects.count() data['n_maintainers'] = Maintainer.objects.count()
data['last_scan'] = EuscanResult.objects.get(id=EuscanResult.objects.aggregate(Max('id'))['id__max']).datetime data['last_scan'] = EuscanResult.objects.get(
id=EuscanResult.objects.aggregate(Max('id'))['id__max']
).datetime
return data return data
# /api/1.0/maintainers # /api/1.0/maintainers
class MaintainersHandler(AnonymousBaseHandler): class MaintainersHandler(AnonymousBaseHandler):
allowed_methods = ('GET',) allowed_methods = ('GET',)
def read(self, request): def read(self, request):
maintainers = Package.objects.filter(maintainers__isnull=False) maintainers = Package.objects.filter(maintainers__isnull=False)
maintainers = maintainers.values('maintainers__id', 'maintainers__name', 'maintainers__email') maintainers = maintainers.values(
'maintainers__id',
'maintainers__name',
'maintainers__email'
)
maintainers = maintainers.annotate(n_packaged=Sum('n_packaged'), maintainers = maintainers.annotate(n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'), n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')) n_versions=Sum('n_versions'))
maintainers = renameFields(maintainers, [('maintainers__id', 'id'), maintainers = renameFields(
('maintainers__name', 'name'), maintainers,
('maintainers__email', 'email')]) [('maintainers__id', 'id'),
return { 'maintainers' : maintainers } ('maintainers__name', 'name'),
('maintainers__email', 'email')]
)
return {'maintainers': maintainers}
# /api/1.0/herds # /api/1.0/herds
class HerdsHandler(AnonymousBaseHandler): class HerdsHandler(AnonymousBaseHandler):
allowed_methods = ('GET',) allowed_methods = ('GET',)
def read(self, request): def read(self, request):
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of 'INNER JOIN' # FIXME: optimize the query, it uses 'LEFT OUTER JOIN'
# instead of 'INNER JOIN'
herds = Package.objects.filter(herds__isnull=False) herds = Package.objects.filter(herds__isnull=False)
herds = herds.values('herds__herd').annotate(n_packaged=Sum('n_packaged'), herds = herds.values('herds__herd').annotate(
n_overlay=Sum('n_overlay'), n_packaged=Sum('n_packaged'),
n_versions=Sum('n_versions')) n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')
)
herds = renameFields(herds, [('herds__herd', 'herd')]) herds = renameFields(herds, [('herds__herd', 'herd')])
return { 'herds' : herds } return {'herds': herds}
# /api/1.0/categories # /api/1.0/categories
class CategoriesHandler(AnonymousBaseHandler): class CategoriesHandler(AnonymousBaseHandler):
@ -106,7 +128,8 @@ class CategoriesHandler(AnonymousBaseHandler):
n_overlay=Sum('n_overlay'), n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')) n_versions=Sum('n_versions'))
return { 'categories' : categories } return {'categories': categories}
# /api/1.0/packages/by-maintainer/ # /api/1.0/packages/by-maintainer/
# /api/1.0/packages/by-category/ # /api/1.0/packages/by-category/
@ -125,17 +148,21 @@ class PackagesHandler(AnonymousBaseHandler):
if 'category' in kwargs: if 'category' in kwargs:
packages = Package.objects.filter(category=kwargs['category']) packages = Package.objects.filter(category=kwargs['category'])
data = { 'category' : kwargs['category'] } data = {'category': kwargs['category']}
elif 'herd' in kwargs: elif 'herd' in kwargs:
herd = Herd.objects.get(herd=kwargs['herd']) herd = Herd.objects.get(herd=kwargs['herd'])
packages = Package.objects.filter(herds__id=herd.id) packages = Package.objects.filter(herds__id=herd.id)
data = { 'herd' : herd } data = {'herd': herd}
elif 'maintainer_id' in kwargs: elif 'maintainer_id' in kwargs:
maintainer = Maintainer.objects.get(id=kwargs['maintainer_id']) maintainer = Maintainer.objects.get(id=kwargs['maintainer_id'])
packages = Package.objects.filter(maintainers__id=maintainer.id) packages = Package.objects.filter(maintainers__id=maintainer.id)
data = { 'maintainer' : maintainer } data = {'maintainer': maintainer}
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream') packages = packages.select_related(
'last_version_gentoo',
'last_version_overlay',
'last_version_upstream'
)
data['packages'] = packages data['packages'] = packages
if not data: if not data:
@ -143,6 +170,7 @@ class PackagesHandler(AnonymousBaseHandler):
return data return data
# /api/1.0/package/ # /api/1.0/package/
class PackageHandler(AnonymousBaseHandler): class PackageHandler(AnonymousBaseHandler):
allowed_methods = ('GET',) allowed_methods = ('GET',)
@ -152,7 +180,8 @@ class PackageHandler(AnonymousBaseHandler):
package = Package.objects.get(category=category, name=package) package = Package.objects.get(category=category, name=package)
package.homepages = package.homepage.split(' ') package.homepages = package.homepage.split(' ')
versions = Version.objects.filter(package=package) versions = Version.objects.filter(package=package)
log = EuscanResult.objects.filter(package=package).order_by('-datetime')[:1] log = EuscanResult.objects.filter(package=package).\
order_by('-datetime')[:1]
log = log[0] if log else None log = log[0] if log else None
vlog = VersionLog.objects.filter(package=package).order_by('-id') vlog = VersionLog.objects.filter(package=package).order_by('-id')
@ -166,7 +195,8 @@ class PackageHandler(AnonymousBaseHandler):
version_log = [] version_log = []
for v in vlog: for v in vlog:
v = model_to_dict(v, ['version', 'revision', 'slot', 'overlay', 'datetime', 'action']) v = model_to_dict(v, ['version', 'revision', 'slot', 'overlay',
'datetime', 'action'])
if v['action'] == VersionLog.VERSION_ADDED: if v['action'] == VersionLog.VERSION_ADDED:
v['action'] = 'added' v['action'] = 'added'
if v['action'] == VersionLog.VERSION_REMOVED: if v['action'] == VersionLog.VERSION_REMOVED:
@ -177,7 +207,10 @@ class PackageHandler(AnonymousBaseHandler):
packaged = [] packaged = []
for version in versions: for version in versions:
unpackaged = not version.packaged unpackaged = not version.packaged
version = model_to_dict(version, ['version', 'revision', 'slot', 'overlay', 'urls']) version = model_to_dict(
version,
['version', 'revision', 'slot', 'overlay', 'urls']
)
if unpackaged: if unpackaged:
upstream.append(version) upstream.append(version)
else: else:

32
euscanwww/djeuscan/api/urls.py
View File

@ -1,7 +1,8 @@
from django.conf.urls.defaults import * from django.conf.urls.defaults import url, patterns
from piston.resource import Resource from piston.resource import Resource
from handlers import * from handlers import RootHandler, StatisticsHandler, HerdsHandler, \
CategoriesHandler, MaintainersHandler, PackagesHandler, PackageHandler
root_handler = Resource(handler=RootHandler) root_handler = Resource(handler=RootHandler)
statistics_handler = Resource(handler=StatisticsHandler) statistics_handler = Resource(handler=StatisticsHandler)
@ -12,16 +13,25 @@ packages_handler = Resource(handler=PackagesHandler)
package_handler = Resource(handler=PackageHandler) package_handler = Resource(handler=PackageHandler)
urlpatterns = patterns('djeuscan.api.views', urlpatterns = patterns('djeuscan.api.views',
url(r'^1.0/statistics\.(?P<emitter_format>.+)$', statistics_handler, name='api.views.statistics'), url(r'^1.0/statistics\.(?P<emitter_format>.+)$', statistics_handler,
url(r'^1.0/herds\.(?P<emitter_format>.+)$', herds_handler, name='api.views.herds'), name='api.views.statistics'),
url(r'^1.0/categories\.(?P<emitter_format>.+)$', categories_handler, name='api.views.categories'), url(r'^1.0/herds\.(?P<emitter_format>.+)$', herds_handler,
url(r'^1.0/maintainers\.(?P<emitter_format>.+)$', maintainers_handler, name='api.views.maintainers'), name='api.views.herds'),
url(r'^1.0/categories\.(?P<emitter_format>.+)$', categories_handler,
name='api.views.categories'),
url(r'^1.0/maintainers\.(?P<emitter_format>.+)$', maintainers_handler,
name='api.views.maintainers'),
url(r'^1.0/packages/by-maintainer/(?P<maintainer_id>\d+)\.(?P<emitter_format>.+)$', packages_handler, name='api.views.packages'), url(r'^1.0/packages/by-maintainer/(?P<maintainer_id>\d+)\.(?P<emitter_format>.+)$',
url(r'^1.0/packages/by-herd/(?P<herd>[\@\{\}\w+.-]*)\.(?P<emitter_format>.+)?$', packages_handler, name='api.views.packages'), packages_handler, name='api.views.packages'),
url(r'^1.0/packages/by-category/(?P<category>[\w+][\w+.-]*)\.(?P<emitter_format>.+)?$', packages_handler, name='api.views.packages'), url(r'^1.0/packages/by-herd/(?P<herd>[\@\{\}\w+.-]*)\.(?P<emitter_format>.+)?$',
packages_handler, name='api.views.packages'),
url(r'^1.0/packages/by-category/(?P<category>[\w+][\w+.-]*)\.(?P<emitter_format>.+)?$',
packages_handler, name='api.views.packages'),
url(r'^1.0/package/(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)\.(?P<emitter_format>.+)$', package_handler, name='api.views.package'), url(r'^1.0/package/(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)\.(?P<emitter_format>.+)$',
package_handler, name='api.views.package'),
url(r'^1.0/api\.(?P<emitter_format>.+)$', root_handler, name='api.views.root'), url(r'^1.0/api\.(?P<emitter_format>.+)$',
root_handler, name='api.views.root'),
) )

147
euscanwww/djeuscan/charts.py
View File

@ -3,14 +3,12 @@ import time
from euscanwww import settings from euscanwww import settings
from django.db.models import F, Sum, Max from django.db.models import F, Sum
from djeuscan.models import Version, Package, Herd, Maintainer from djeuscan.models import Package
from djeuscan.models import CategoryLog
import rrdtool import rrdtool
import pylab import pylab
import matplotlib
CHARTS_ROOT = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts') CHARTS_ROOT = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts')
CHARTS_URL = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts') CHARTS_URL = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts')
@ -21,12 +19,14 @@ pylab.rcParams['axes.titlesize'] = 10.0
pylab.rcParams['xtick.labelsize'] = 8.0 pylab.rcParams['xtick.labelsize'] = 8.0
pylab.rcParams['legend.fontsize'] = 8.0 pylab.rcParams['legend.fontsize'] = 8.0
def xint(i): def xint(i):
try: try:
return int(i) return int(i)
except: except:
return 0 return 0
def chart_alive(name): def chart_alive(name):
path = os.path.join(CHARTS_ROOT, name) path = os.path.join(CHARTS_ROOT, name)
if not os.path.exists(path): if not os.path.exists(path):
@ -37,6 +37,7 @@ def chart_alive(name):
return False return False
return True return True
def rrd_name(**kwargs): def rrd_name(**kwargs):
name = "" name = ""
@ -51,6 +52,7 @@ def rrd_name(**kwargs):
return name return name
def chart_name(name, **kwargs): def chart_name(name, **kwargs):
name = name.replace('_', '-') name = name.replace('_', '-')
@ -67,6 +69,7 @@ def chart_name(name, **kwargs):
return name + ".png" return name + ".png"
def getpackages(**kwargs): def getpackages(**kwargs):
packages = Package.objects packages = Package.objects
@ -79,6 +82,7 @@ def getpackages(**kwargs):
return packages return packages
def cached_pylab_chart(f): def cached_pylab_chart(f):
def new_f(*args, **kwds): def new_f(*args, **kwds):
name = chart_name(f.func_name, **kwds) name = chart_name(f.func_name, **kwds)
@ -93,14 +97,16 @@ def cached_pylab_chart(f):
new_f.func_name = f.func_name new_f.func_name = f.func_name
return new_f return new_f
@cached_pylab_chart @cached_pylab_chart
def pie_versions(**kwargs): def pie_versions(**kwargs):
n_packaged = xint(getpackages(**kwargs).aggregate(Sum('n_packaged'))['n_packaged__sum']) gpk = getpackages(**kwargs)
n_overlay = xint(getpackages(**kwargs).aggregate(Sum('n_overlay'))['n_overlay__sum']) n_packaged = xint(gpk.aggregate(Sum('n_packaged'))['n_packaged__sum'])
n_versions = xint(getpackages(**kwargs).aggregate(Sum('n_versions'))['n_versions__sum']) n_overlay = xint(gpk.aggregate(Sum('n_overlay'))['n_overlay__sum'])
n_versions = xint(gpk.aggregate(Sum('n_versions'))['n_versions__sum'])
n_upstream = n_versions - n_packaged - n_overlay n_upstream = n_versions - n_packaged - n_overlay
pylab.figure(1, figsize=(3.5,3.5)) pylab.figure(1, figsize=(3.5, 3.5))
if n_overlay: if n_overlay:
labels = 'Gentoo', 'Overlays', 'Upstream' labels = 'Gentoo', 'Overlays', 'Upstream'
@ -111,35 +117,43 @@ def pie_versions(**kwargs):
fracs = [n_packaged, n_upstream] fracs = [n_packaged, n_upstream]
colors = '#008000', '#FF0000' colors = '#008000', '#FF0000'
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', shadow=True) pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%',
pylab.title('Versions', bbox={'facecolor':'0.8', 'pad':5}) shadow=True)
pylab.title('Versions', bbox={'facecolor': '0.8', 'pad': 5})
@cached_pylab_chart @cached_pylab_chart
def pie_packages(**kwargs): def pie_packages(**kwargs):
n_packages = getpackages(**kwargs).count() gpk = getpackages(**kwargs)
n_packages_uptodate_main = getpackages(**kwargs).filter(n_versions=F('n_packaged')).count() n_packages = gpk.count()
n_packages_uptodate_all = getpackages(**kwargs).filter(n_versions=F('n_packaged') + F('n_overlay')).count() n_packages_uptodate_main = gpk.filter(n_versions=F('n_packaged')).count()
n_packages_uptodate_all = gpk.filter(n_versions=F('n_packaged') + \
F('n_overlay')).count()
n_packages_outdated = n_packages - n_packages_uptodate_all n_packages_outdated = n_packages - n_packages_uptodate_all
n_packages_uptodate_ovl = n_packages_uptodate_all - n_packages_uptodate_main n_packages_uptodate_ovl = n_packages_uptodate_all - \
n_packages_uptodate_main
pylab.figure(1, figsize=(3.5,3.5)) pylab.figure(1, figsize=(3.5, 3.5))
if n_packages_uptodate_ovl: if n_packages_uptodate_ovl:
labels = 'Ok (gentoo)', 'Ok (overlays)', 'Outdated' labels = 'Ok (gentoo)', 'Ok (overlays)', 'Outdated'
fracs = [n_packages_uptodate_main, n_packages_uptodate_ovl, n_packages_outdated] fracs = [n_packages_uptodate_main, n_packages_uptodate_ovl,
n_packages_outdated]
colors = '#008000', '#0B17FD', '#FF0000' colors = '#008000', '#0B17FD', '#FF0000'
else: else:
labels = 'Ok (gentoo)', 'Outdated' labels = 'Ok (gentoo)', 'Outdated'
fracs = [n_packages_uptodate_main, n_packages_outdated] fracs = [n_packages_uptodate_main, n_packages_outdated]
colors = '#008000', '#FF0000' colors = '#008000', '#FF0000'
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', shadow=True) pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%',
pylab.title('Packages', bbox={'facecolor':'0.8', 'pad':5}) shadow=True)
pylab.title('Packages', bbox={'facecolor': '0.8', 'pad': 5})
def rrd_path(name): def rrd_path(name):
return str(os.path.join(settings.RRD_ROOT, name + '.rrd')) return str(os.path.join(settings.RRD_ROOT, name + '.rrd'))
def rrd_create(name, start): def rrd_create(name, start):
path = rrd_path(name) path = rrd_path(name)
if os.path.exists(path): if os.path.exists(path):
@ -156,13 +170,17 @@ def rrd_create(name, start):
'RRA:AVERAGE:0.5:5:200', 'RRA:AVERAGE:0.5:5:200',
'RRA:AVERAGE:0.5:10:200') 'RRA:AVERAGE:0.5:10:200')
def rrd_update(name, datetime, values): def rrd_update(name, datetime, values):
now = time.mktime(datetime.timetuple()) now = time.mktime(datetime.timetuple())
rrd_create(name, now) rrd_create(name, now)
rrdtool.update(rrd_path(name), rrdtool.update(
'%d:%d:%d:%d:%d:%d:%d' % \ rrd_path(name),
(now, values.n_packages_gentoo, values.n_packages_overlay, values.n_packages_outdated, \ '%d:%d:%d:%d:%d:%d:%d' % \
values.n_versions_gentoo, values.n_versions_overlay, values.n_versions_upstream)) (now, values.n_packages_gentoo, values.n_packages_overlay,
values.n_packages_outdated, values.n_versions_gentoo,
values.n_versions_overlay, values.n_versions_upstream)
)
""" """
@ -175,6 +193,7 @@ def rrd_update(name, datetime, values):
[-J|--alt-autoscale-min] [-J|--alt-autoscale-min]
""" """
def cached_rrd_chart(f): def cached_rrd_chart(f):
def new_f(*args, **kwds): def new_f(*args, **kwds):
if 'period' not in kwds: if 'period' not in kwds:
@ -215,47 +234,55 @@ def cached_rrd_chart(f):
new_f.func_name = f.func_name new_f.func_name = f.func_name
return new_f return new_f
@cached_rrd_chart @cached_rrd_chart
def packages(**kwargs): def packages(**kwargs):
rrdtool.graph(str(kwargs['path']), rrdtool.graph(
'--imgformat', 'PNG', str(kwargs['path']),
'--width', kwargs['width'], '--imgformat', 'PNG',
'--height', kwargs['height'], '--width', kwargs['width'],
kwargs['graph-mode'], '--height', kwargs['height'],
'--color', 'CANVAS#FFFFFF00', kwargs['graph-mode'],
'--color', 'BACK#FFFFFF00', '--color', 'CANVAS#FFFFFF00',
'--color', 'BACK#FFFFFF00',
'--start', kwargs['start'],
'--end', kwargs['end'],
'--vertical-label', kwargs['vertical-label'],
'--title', kwargs['title'],
'--lower-limit', '0',
'DEF:n_packages_gentoo=%s:n_packages_gentoo:AVERAGE' % (kwargs['rrd']),
'DEF:n_packages_overlay=%s:n_packages_overlay:AVERAGE' % \
(kwargs['rrd']),
'DEF:n_packages_outdated=%s:n_packages_outdated:AVERAGE' % \
(kwargs['rrd']),
'LINE1.25:n_packages_gentoo#008000:Gentoo',
'LINE1.25:n_packages_overlay#0B17FD:Overlay',
'LINE1.25:n_packages_outdated#FF0000:Outdated'
)
'--start', kwargs['start'],
'--end', kwargs['end'],
'--vertical-label', kwargs['vertical-label'],
'--title', kwargs['title'],
'--lower-limit', '0',
'DEF:n_packages_gentoo=%s:n_packages_gentoo:AVERAGE' % (kwargs['rrd']),
'DEF:n_packages_overlay=%s:n_packages_overlay:AVERAGE' % (kwargs['rrd']),
'DEF:n_packages_outdated=%s:n_packages_outdated:AVERAGE' % (kwargs['rrd']),
'LINE1.25:n_packages_gentoo#008000:Gentoo',
'LINE1.25:n_packages_overlay#0B17FD:Overlay',
'LINE1.25:n_packages_outdated#FF0000:Outdated')
@cached_rrd_chart @cached_rrd_chart
def versions(**kwargs): def versions(**kwargs):
rrdtool.graph(str(kwargs['path']), rrdtool.graph(
'--imgformat', 'PNG', str(kwargs['path']),
'--width', kwargs['width'], '--imgformat', 'PNG',
'--height', kwargs['height'], '--width', kwargs['width'],
kwargs['graph-mode'], '--height', kwargs['height'],
'--color', 'CANVAS#FFFFFF00', kwargs['graph-mode'],
'--color', 'BACK#FFFFFF00', '--color', 'CANVAS#FFFFFF00',
'--start', kwargs['start'], '--color', 'BACK#FFFFFF00',
'--end', kwargs['end'], '--start', kwargs['start'],
'--vertical-label', kwargs['vertical-label'], '--end', kwargs['end'],
'--title', kwargs['title'], '--vertical-label', kwargs['vertical-label'],
'--lower-limit', '0', '--title', kwargs['title'],
'DEF:n_versions_gentoo=%s:n_versions_gentoo:AVERAGE' % (kwargs['rrd']), '--lower-limit', '0',
'DEF:n_versions_overlay=%s:n_versions_overlay:AVERAGE' % (kwargs['rrd']), 'DEF:n_versions_gentoo=%s:n_versions_gentoo:AVERAGE' % (kwargs['rrd']),
'DEF:n_versions_outdated=%s:n_versions_upstream:AVERAGE' % (kwargs['rrd']), 'DEF:n_versions_overlay=%s:n_versions_overlay:AVERAGE' % \
'LINE1.25:n_versions_gentoo#008000:Gentoo', (kwargs['rrd']),
'LINE1.25:n_versions_overlay#0B17FD:Overlay', 'DEF:n_versions_outdated=%s:n_versions_upstream:AVERAGE' % \
'LINE1.25:n_versions_outdated#FF0000:Outdated') (kwargs['rrd']),
'LINE1.25:n_versions_gentoo#008000:Gentoo',
'LINE1.25:n_versions_overlay#0B17FD:Overlay',
'LINE1.25:n_versions_outdated#FF0000:Outdated'
)

32
euscanwww/djeuscan/feeds.py
View File

@ -1,11 +1,10 @@
from django.contrib.syndication.views import Feed, FeedDoesNotExist from django.contrib.syndication.views import Feed, FeedDoesNotExist
from django.shortcuts import get_object_or_404 from django.shortcuts import get_object_or_404
from django.http import Http404
from django.utils.feedgenerator import Atom1Feed from django.utils.feedgenerator import Atom1Feed
from django.core.urlresolvers import reverse from django.core.urlresolvers import reverse
from djeuscan.models import Version, Package, Herd, Maintainer, VersionLog from djeuscan.models import Package, Herd, Maintainer, VersionLog
from djeuscan.views import *
class BaseFeed(Feed): class BaseFeed(Feed):
feed_type = Atom1Feed feed_type = Atom1Feed
@ -18,8 +17,8 @@ class BaseFeed(Feed):
def item_description(self, vlog): def item_description(self, vlog):
if vlog.overlay: if vlog.overlay:
txt = 'Version %s-%s [%s] of package %s ' % (vlog.version, vlog.revision, txt = 'Version %s-%s [%s] of package %s ' % \
vlog.slot, vlog.package) (vlog.version, vlog.revision, vlog.slot, vlog.package)
else: else:
txt = 'Version %s of package %s ' % (vlog.version, vlog.package) txt = 'Version %s of package %s ' % (vlog.version, vlog.package)
if vlog.action == vlog.VERSION_REMOVED: if vlog.action == vlog.VERSION_REMOVED:
@ -36,8 +35,10 @@ class BaseFeed(Feed):
return txt return txt
def item_link(self, vlog): def item_link(self, vlog):
kwargs = {'category' : vlog.package.category, 'package' : vlog.package.name } kwargs = {'category': vlog.package.category,
return reverse('djeuscan.views.package', kwargs=kwargs) + '#' + vlog.tag() 'package': vlog.package.name}
return "%s#%s" % (reverse('djeuscan.views.package', kwargs=kwargs),
vlog.tag())
def item_pubdate(self, vlog): def item_pubdate(self, vlog):
return vlog.datetime return vlog.datetime
@ -45,18 +46,20 @@ class BaseFeed(Feed):
def item_categories(self, vlog): def item_categories(self, vlog):
return [vlog.package.category] return [vlog.package.category]
class GlobalFeed(BaseFeed): class GlobalFeed(BaseFeed):
title = "euscan" title = "euscan"
link = "/" link = "/"
description = "Last euscan changes" description = "Last euscan changes"
def categories(self): def categories(self):
categories = Package.objects.values('category').distinct(); categories = Package.objects.values('category').distinct()
return [ category['category'] for category in categories ] return [category['category'] for category in categories]
def items(self): def items(self):
return VersionLog.objects.order_by('-id')[:250] return VersionLog.objects.order_by('-id')[:250]
class PackageFeed(BaseFeed): class PackageFeed(BaseFeed):
feed_type = Atom1Feed feed_type = Atom1Feed
@ -67,7 +70,8 @@ class PackageFeed(BaseFeed):
return "%s" % package return "%s" % package
def link(self, package): def link(self, package):
return reverse('djeuscan.views.package', args=[package.category, package.name]) return reverse('djeuscan.views.package', args=[package.category,
package.name])
def description(self, package): def description(self, package):
return package.description return package.description
@ -78,6 +82,7 @@ class PackageFeed(BaseFeed):
def item_description(self, vlog): def item_description(self, vlog):
return '' return ''
class MaintainerFeed(BaseFeed): class MaintainerFeed(BaseFeed):
feed_type = Atom1Feed feed_type = Atom1Feed
@ -91,12 +96,14 @@ class MaintainerFeed(BaseFeed):
return "Last changes for %s" % maintainer return "Last changes for %s" % maintainer
def link(self, maintainer): def link(self, maintainer):
return reverse('djeuscan.views.maintainer', kwargs={'maintainer_id' : maintainer.id}) return reverse('djeuscan.views.maintainer',
kwargs={'maintainer_id': maintainer.id})
def items(self, maintainer): def items(self, maintainer):
q = VersionLog.objects.filter(package__maintainers__id=maintainer.id) q = VersionLog.objects.filter(package__maintainers__id=maintainer.id)
return q.order_by('-id')[:50] return q.order_by('-id')[:50]
class HerdFeed(BaseFeed): class HerdFeed(BaseFeed):
feed_type = Atom1Feed feed_type = Atom1Feed
@ -110,12 +117,13 @@ class HerdFeed(BaseFeed):
return "Last changes for %s" % herd return "Last changes for %s" % herd
def link(self, herd): def link(self, herd):
return reverse('djeuscan.views.herd', kwargs={'herd' : herd.herd}) return reverse('djeuscan.views.herd', kwargs={'herd': herd.herd})
def items(self, herd): def items(self, herd):
q = VersionLog.objects.filter(package__herds__id=herd.id) q = VersionLog.objects.filter(package__herds__id=herd.id)
return q.order_by('-id')[:100] return q.order_by('-id')[:100]
class CategoryFeed(BaseFeed): class CategoryFeed(BaseFeed):
feed_type = Atom1Feed feed_type = Atom1Feed

2
euscanwww/djeuscan/forms.py
View File

@ -1,7 +1,9 @@
from django import forms from django import forms
class WorldForm(forms.Form): class WorldForm(forms.Form):
world = forms.FileField() world = forms.FileField()
class PackagesForm(forms.Form): class PackagesForm(forms.Form):
packages = forms.CharField(widget=forms.Textarea) packages = forms.CharField(widget=forms.Textarea)

3
euscanwww/djeuscan/management/commands/list-packages.py
View File

@ -1,8 +1,9 @@
from optparse import make_option from optparse import make_option
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand
from djeuscan.models import Package from djeuscan.models import Package
class Command(BaseCommand): class Command(BaseCommand):
_overlays = {} _overlays = {}
help = 'List packages' help = 'List packages'

13
euscanwww/djeuscan/management/commands/regen-rrds.py
View File

@ -1,11 +1,8 @@
import datetime from django.core.management.base import BaseCommand
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from djeuscan import charts from djeuscan import charts
class Command(BaseCommand): class Command(BaseCommand):
_overlays = {} _overlays = {}
help = 'Regenerate rrd database' help = 'Regenerate rrd database'
@ -15,10 +12,12 @@ class Command(BaseCommand):
charts.rrd_update('world', wlog.datetime, wlog) charts.rrd_update('world', wlog.datetime, wlog)
for clog in CategoryLog.objects.all(): for clog in CategoryLog.objects.all():
charts.rrd_update('category-%s' % clog.category, clog.datetime, clog) charts.rrd_update('category-%s' % clog.category,
clog.datetime, clog)
for hlog in HerdLog.objects.all(): for hlog in HerdLog.objects.all():
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog) charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
for mlog in MaintainerLog.objects.all(): for mlog in MaintainerLog.objects.all():
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, mlog.datetime, mlog) charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
mlog.datetime, mlog)

53
euscanwww/djeuscan/management/commands/scan-metadata.py
View File

@ -1,19 +1,15 @@
import subprocess
import portage
import sys import sys
import os
import re
from portage import versions
from optparse import make_option from optparse import make_option
from django.db.transaction import commit_on_success from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand
from djeuscan.models import Package, Herd, Maintainer from djeuscan.models import Package, Herd, Maintainer
from gentoolkit.query import Query from gentoolkit.query import Query
from gentoolkit.errors import GentoolkitFatalError from gentoolkit.errors import GentoolkitFatalError
class Command(BaseCommand): class Command(BaseCommand):
_overlays = {} _overlays = {}
@ -53,16 +49,20 @@ class Command(BaseCommand):
) )
if not matches: if not matches:
sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % query)) sys.stderr.write(
self.style.ERROR("Unknown package '%s'\n" % query)
)
return return
matches = sorted(matches) matches = sorted(matches)
pkg = matches.pop() pkg = matches.pop()
if '9999' in pkg.version and len(matches): if '9999' in pkg.version and len(matches):
pkg = matches.pop() pkg = matches.pop()
if not obj: if not obj:
obj, created = Package.objects.get_or_create(category=pkg.category, name=pkg.name) obj, created = Package.objects.get_or_create(
category=pkg.category, name=pkg.name
)
else: else:
created = False created = False
@ -70,22 +70,34 @@ class Command(BaseCommand):
obj.homepage = pkg.environment("HOMEPAGE") obj.homepage = pkg.environment("HOMEPAGE")
obj.description = pkg.environment("DESCRIPTION") obj.description = pkg.environment("DESCRIPTION")
except GentoolkitFatalError, err: except GentoolkitFatalError, err:
sys.stderr.write(self.style.ERROR("Gentoolkit fatal error: '%s'\n" % str(err))) sys.stderr.write(
self.style.ERROR(
"Gentoolkit fatal error: '%s'\n" % str(err)
)
)
if created and not self.options['quiet']: if created and not self.options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name)) sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
if pkg.metadata: if pkg.metadata:
herds = dict([(herd[0], herd) for herd in pkg.metadata.herds(True)]) herds = dict(
maintainers = dict([(m.email, m) for m in pkg.metadata.maintainers()]) [(herd[0], herd) for herd in pkg.metadata.herds(True)]
)
maintainers = dict(
[(m.email, m) for m in pkg.metadata.maintainers()]
)
existing_herds = [h.herd for h in obj.herds.all()] existing_herds = [h.herd for h in obj.herds.all()]
new_herds = set(herds.keys()).difference(existing_herds) new_herds = set(herds.keys()).difference(existing_herds)
old_herds = set(existing_herds).difference(herds.keys()) old_herds = set(existing_herds).difference(herds.keys())
existing_maintainers = [m.email for m in obj.maintainers.all()] existing_maintainers = [m.email for m in obj.maintainers.all()]
new_maintainers = set(maintainers.keys()).difference(existing_maintainers) new_maintainers = set(
old_maintainers = set(existing_maintainers).difference(maintainers.keys()) maintainers.keys()).difference(existing_maintainers
)
old_maintainers = set(
existing_maintainers).difference(maintainers.keys()
)
for herd in obj.herds.all(): for herd in obj.herds.all():
if herd.herd in old_herds: if herd.herd in old_herds:
@ -101,7 +113,9 @@ class Command(BaseCommand):
for maintainer in new_maintainers: for maintainer in new_maintainers:
maintainer = maintainers[maintainer] maintainer = maintainers[maintainer]
maintainer = self.store_maintainer(maintainer.name, maintainer.email) maintainer = self.store_maintainer(
maintainer.name, maintainer.email
)
obj.maintainers.add(maintainer) obj.maintainers.add(maintainer)
obj.save() obj.save()
@ -131,9 +145,12 @@ class Command(BaseCommand):
if created: if created:
if not self.options['quiet']: if not self.options['quiet']:
sys.stdout.write('+ [m] %s <%s>\n' % (name.encode('utf-8'), email)) sys.stdout.write(
'+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
)
if not maintainer.name or name not in [maintainer.name, email, '{nil}']: if not maintainer.name or \
name not in [maintainer.name, email, '{nil}']:
maintainer.name = name maintainer.name = name
maintainer.save() maintainer.save()

94
euscanwww/djeuscan/management/commands/scan-portage.py
View File

@ -4,13 +4,13 @@ import sys
import os import os
import re import re
from portage import versions
from optparse import make_option from optparse import make_option
from django.db.transaction import commit_on_success from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand
from djeuscan.models import Package, Version, VersionLog from djeuscan.models import Package, Version, VersionLog
class Command(BaseCommand): class Command(BaseCommand):
_overlays = {} _overlays = {}
@ -39,7 +39,8 @@ class Command(BaseCommand):
action='store_true', action='store_true',
dest='prefetch', dest='prefetch',
default=False, default=False,
help='Prefetch all versions and packages from DB to speedup full scan process.'), help=('Prefetch all versions and packages from DB to '
'speedup full scan process.')),
make_option('--quiet', make_option('--quiet',
action='store_true', action='store_true',
dest='quiet', dest='quiet',
@ -49,7 +50,7 @@ class Command(BaseCommand):
args = '[package package ...]' args = '[package package ...]'
help = 'Scans portage tree and fills database' help = 'Scans portage tree and fills database'
_cache = {'packages' : {}, 'versions' : {}} _cache = {'packages': {}, 'versions': {}}
def cache_hash_package(self, category, name): def cache_hash_package(self, category, name):
return '%s/%s' % (category, name) return '%s/%s' % (category, name)
@ -59,22 +60,28 @@ class Command(BaseCommand):
self._cache['packages'][key] = package self._cache['packages'][key] = package
def cache_get_package(self, category, name): def cache_get_package(self, category, name):
return self._cache['packages'].get(self.cache_hash_package(category, name)) return self._cache['packages'].get(
self.cache_hash_package(category, name)
)
def cache_hash_version(self, category, name, version, revision, slot, overlay): def cache_hash_version(self, category, name, version, revision, slot,
overlay):
key = '%s/%s-%s-r%s %s %s' % (category, name, key = '%s/%s-%s-r%s %s %s' % (category, name,
version, revision, version, revision,
slot, overlay) slot, overlay)
return key return key
def cache_get_version(self, category, name, version, revision, slot, overlay): def cache_get_version(self, category, name, version, revision, slot,
key = self.cache_hash_version(category, name, version, revision, slot, overlay) overlay):
key = self.cache_hash_version(category, name, version, revision, slot,
overlay)
return self._cache['versions'].get(key) return self._cache['versions'].get(key)
def cache_store_version(self, version): def cache_store_version(self, version):
key = self.cache_hash_version(version.package.category, version.package.name, key = self.cache_hash_version(
version.version, version.revision, version.slot, version.package.category, version.package.name, version.version,
version.overlay) version.revision, version.slot, version.overlay
)
self._cache['versions'][key] = version self._cache['versions'][key] = version
def handle(self, *args, **options): def handle(self, *args, **options):
@ -119,7 +126,8 @@ class Command(BaseCommand):
cmd = ['eix', '-!'] cmd = ['eix', '-!']
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0] output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
output = output.strip().strip('\n').split('\n') output = output.strip().strip('\n').split('\n')
overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"') overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
@ -139,9 +147,10 @@ class Command(BaseCommand):
env = os.environ env = os.environ
env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n" env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages', '-x'] cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages',
if query: '-x']
cmd.extend(['--exact', query]) if query:
cmd.extend(['--exact', query])
if self.options['all']: if self.options['all']:
if not self.options['quiet']: if not self.options['quiet']:
@ -151,7 +160,8 @@ class Command(BaseCommand):
if not self.options['quiet']: if not self.options['quiet']:
self.stdout.write('done\n') self.stdout.write('done\n')
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0] output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
output = output.strip().strip('\n') output = output.strip().strip('\n')
if len(output) == 0: if len(output) == 0:
@ -166,17 +176,23 @@ class Command(BaseCommand):
else: else:
Package.objects.filter(name=query).delete() Package.objects.filter(name=query).delete()
else: else:
sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % query)) sys.stderr.write(
self.style.ERROR(
"Unknown package '%s'\n" % query
)
)
return return
output = output.split('\n') output = output.split('\n')
packages = {} packages = {}
line_re = re.compile(r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$') line_re = re.compile(
r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$'
)
package = None package = None
for line in output: for line in output:
match = line_re.match(line) match = line_re.match(line)
if not match: if not match:
@ -190,7 +206,8 @@ class Command(BaseCommand):
packages['%s/%s' % (cat, pkg)] = True packages['%s/%s' % (cat, pkg)] = True
if not package or not (cat == package.category and pkg == package.name): if not package or not \
(cat == package.category and pkg == package.name):
package = self.store_package(cat, pkg) package = self.store_package(cat, pkg)
self.store_version(package, cpv, slot, overlay) self.store_version(package, cpv, slot, overlay)
@ -208,16 +225,23 @@ class Command(BaseCommand):
obj = self.cache_get_package(cat, pkg) obj = self.cache_get_package(cat, pkg)
if not obj: if not obj:
obj, created = Package.objects.get_or_create(category=cat, name=pkg) obj, created = Package.objects.get_or_create(
category=cat,
name=pkg
)
self.cache_store_package(obj) self.cache_store_package(obj)
if created: if created:
if not self.options['quiet']: if not self.options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg)) sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
' Set all versions dead, then set found versions alive and delete old versions ' # Set all versions dead, then set found versions alive and
# delete old versions
if not self.options['all']: if not self.options['all']:
Version.objects.filter(package=obj, packaged=True).update(alive=False) Version.objects.filter(
package=obj,
packaged=True
).update(alive=False)
return obj return obj
@ -232,11 +256,15 @@ class Command(BaseCommand):
overlay = 'gentoo' overlay = 'gentoo'
created = False created = False
obj = self.cache_get_version(package.category, package.name, ver, rev, slot, overlay) obj = self.cache_get_version(
package.category, package.name, ver, rev, slot, overlay
)
if not obj: if not obj:
obj, created = Version.objects.get_or_create(package=package, slot=slot, obj, created = Version.objects.get_or_create(
revision=rev, version=ver, package=package, slot=slot,
overlay=overlay) revision=rev, version=ver,
overlay=overlay
)
obj.alive = True obj.alive = True
obj.packaged = True obj.packaged = True
@ -245,7 +273,8 @@ class Command(BaseCommand):
if created: if created:
self.cache_store_version(obj) self.cache_store_version(obj)
''' nothing to do (note: it can't be an upstream version because overlay can't be empty here) ''' # nothing to do (note: it can't be an upstream version because
# overlay can't be empty here)
if not created: if not created:
return return
@ -287,8 +316,10 @@ class Command(BaseCommand):
if self.options['no-log']: if self.options['no-log']:
continue continue
entry = VersionLog.objects.create(package=version.package, entry = VersionLog.objects.create(
action=VersionLog.VERSION_REMOVED) package=version.package,
action=VersionLog.VERSION_REMOVED
)
entry.slot = version.slot entry.slot = version.slot
entry.revision = version.revision entry.revision = version.revision
entry.version = version.version entry.version = version.version
@ -296,4 +327,3 @@ class Command(BaseCommand):
entry.save() entry.save()
Version.objects.filter(packaged=True, alive=False).delete() Version.objects.filter(packaged=True, alive=False).delete()

38
euscanwww/djeuscan/management/commands/scan-upstream.py
View File

@ -1,19 +1,17 @@
import subprocess import subprocess
import portage import portage
import sys import sys
import os
import re import re
from StringIO import StringIO from StringIO import StringIO
from datetime import datetime
from portage import versions
from optparse import make_option from optparse import make_option
from django.utils import timezone from django.utils import timezone
from django.db.transaction import commit_on_success from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand
from djeuscan.models import Package, Version, EuscanResult, VersionLog from djeuscan.models import Package, Version, EuscanResult, VersionLog
class Command(BaseCommand): class Command(BaseCommand):
_overlays = {} _overlays = {}
@ -60,7 +58,7 @@ class Command(BaseCommand):
elif args: elif args:
packages = list(args) packages = list(args)
else: else:
packages = [ package[:-1] for package in sys.stdin.readlines() ] packages = [package[:-1] for package in sys.stdin.readlines()]
self.scan(options, packages) self.scan(options, packages)
@ -83,8 +81,12 @@ class Command(BaseCommand):
def parse_output(self, options, output): def parse_output(self, options, output):
from portage.versions import _cp from portage.versions import _cp
package_re = re.compile(r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$') package_re = re.compile(
version_re = re.compile(r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$') r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$'
)
version_re = re.compile(
r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$'
)
package = None package = None
log = "" log = ""
@ -124,7 +126,6 @@ class Command(BaseCommand):
obj.datetime = timezone.now() obj.datetime = timezone.now()
obj.save() obj.save()
def store_package(self, options, cpv): def store_package(self, options, cpv):
cat, pkg, ver, rev = portage.catpkgsplit(cpv) cat, pkg, ver, rev = portage.catpkgsplit(cpv)
@ -133,15 +134,16 @@ class Command(BaseCommand):
if created and not options['quiet']: if created and not options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg)) sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
' Set all versions dead, then set found versions alive and delete old versions ' # Set all versions dead, then set found versions alive and
# delete old versions
Version.objects.filter(package=obj, packaged=False).update(alive=False) Version.objects.filter(package=obj, packaged=False).update(alive=False)
return obj return obj
def store_version(self, options, package, ver, url): def store_version(self, options, package, ver, url):
obj, created = Version.objects.get_or_create(package=package, slot='', obj, created = Version.objects.get_or_create(
revision='r0', version=ver, package=package, slot='', revision='r0', version=ver, overlay=''
overlay='') )
obj.alive = True obj.alive = True
obj.urls = url obj.urls = url
@ -155,7 +157,10 @@ class Command(BaseCommand):
if not options['quiet']: if not options['quiet']:
sys.stdout.write('+ [u] %s %s\n' % (obj, url)) sys.stdout.write('+ [u] %s %s\n' % (obj, url))
entry = VersionLog.objects.create(package=package, action=VersionLog.VERSION_ADDED) entry = VersionLog.objects.create(
package=package,
action=VersionLog.VERSION_ADDED
)
entry.slot = '' entry.slot = ''
entry.revision = 'r0' entry.revision = 'r0'
entry.version = ver entry.version = ver
@ -165,12 +170,14 @@ class Command(BaseCommand):
package.n_versions += 1 package.n_versions += 1
package.save() package.save()
@commit_on_success @commit_on_success
def purge_versions(self, options): def purge_versions(self, options):
' For each dead versions ' ' For each dead versions '
for version in Version.objects.filter(packaged=False, alive=False): for version in Version.objects.filter(packaged=False, alive=False):
entry = VersionLog.objects.create(package=version.package, action=VersionLog.VERSION_REMOVED) entry = VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED
)
entry.slot = version.slot entry.slot = version.slot
entry.revision = version.revision entry.revision = version.revision
entry.version = version.version entry.version = version.version
@ -183,4 +190,3 @@ class Command(BaseCommand):
if not options['quiet']: if not options['quiet']:
sys.stdout.write('- [u] %s %s\n' % (version, version.urls)) sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
Version.objects.filter(packaged=False, alive=False).delete() Version.objects.filter(packaged=False, alive=False).delete()

55
euscanwww/djeuscan/management/commands/update-counters.py
View File

@ -1,10 +1,7 @@
import datetime
from optparse import make_option from optparse import make_option
from django.db.models import Count, Sum
from django.db.transaction import commit_on_success from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand
from django.utils import timezone from django.utils import timezone
from djeuscan.models import Package, Herd, Maintainer, Version from djeuscan.models import Package, Herd, Maintainer, Version
@ -13,6 +10,7 @@ from djeuscan import charts
from distutils.version import StrictVersion, LooseVersion from distutils.version import StrictVersion, LooseVersion
def compare_versions(version1, version2): def compare_versions(version1, version2):
try: try:
return cmp(StrictVersion(version1), StrictVersion(version2)) return cmp(StrictVersion(version1), StrictVersion(version2))
@ -20,6 +18,7 @@ def compare_versions(version1, version2):
except ValueError: except ValueError:
return cmp(LooseVersion(version1), LooseVersion(version2)) return cmp(LooseVersion(version1), LooseVersion(version2))
class Command(BaseCommand): class Command(BaseCommand):
_overlays = {} _overlays = {}
help = 'Update counters' help = 'Update counters'
@ -98,7 +97,8 @@ class Command(BaseCommand):
return return
if version['version'].startswith('9999'): if version['version'].startswith('9999'):
return return
if compare_versions(storage[key]['version'], version['version']) < 0: if compare_versions(storage[key]['version'],
version['version']) < 0:
storage[key] = version storage[key] = version
if not options['fast']: if not options['fast']:
@ -125,25 +125,36 @@ class Command(BaseCommand):
package.n_packaged = n_packaged.get(package.id, 0) package.n_packaged = n_packaged.get(package.id, 0)
package.n_overlay = n_overlay.get(package.id, 0) package.n_overlay = n_overlay.get(package.id, 0)
default = {'id' : None} default = {'id': None}
package.last_version_gentoo_id = last_versions_gentoo.get(package.id, default)['id'] package.last_version_gentoo_id = last_versions_gentoo.get(
package.last_version_overlay_id = last_versions_overlay.get(package.id, default)['id'] package.id, default
package.last_version_upstream_id = last_versions_upstream.get(package.id, default)['id'] )['id']
package.last_version_overlay_id = last_versions_overlay.get(
package.id, default
)['id']
package.last_version_upstream_id = last_versions_upstream.get(
package.id, default
)['id']
package.save() package.save()
n_packages_gentoo = int(package.n_packaged == package.n_versions) n_packages_gentoo = int(package.n_packaged == package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged + package.n_overlay == package.n_versions) n_packages_overlay = int(package.n_overlay and package.n_packaged \
n_packages_outdated = int(package.n_packaged + package.n_overlay < package.n_versions) + package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay \
< package.n_versions)
def update_row(storage, key): def update_row(storage, key):
storage[key].n_packages_gentoo += n_packages_gentoo storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_outdated += n_packages_outdated storage[key].n_packages_outdated += n_packages_outdated
storage[key].n_versions_gentoo += package.n_packaged storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay storage[key].n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
def update_log(storage, qs): def update_log(storage, qs):
for row in qs: for row in qs:
update_row(storage, row['id']) update_row(storage, row['id'])
@ -153,13 +164,15 @@ class Command(BaseCommand):
update_log(maintainers, package.maintainers.all().values('id')) update_log(maintainers, package.maintainers.all().values('id'))
update_row(categories, package.category) update_row(categories, package.category)
wlog.n_packages_gentoo += n_packages_gentoo wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated wlog.n_packages_outdated += n_packages_outdated
wlog.n_versions_gentoo += package.n_packaged wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay wlog.n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
if options['nolog']: if options['nolog']:
return return

64
euscanwww/djeuscan/models.py
View File

@ -1,5 +1,6 @@
from django.db import models from django.db import models
class Herd(models.Model): class Herd(models.Model):
herd = models.CharField(max_length=128, unique=True) herd = models.CharField(max_length=128, unique=True)
email = models.CharField(max_length=128, blank=True, null=True) email = models.CharField(max_length=128, blank=True, null=True)
@ -9,6 +10,7 @@ class Herd(models.Model):
return '%s <%s>' % (self.herd, self.email) return '%s <%s>' % (self.herd, self.email)
return self.herd return self.herd
class Maintainer(models.Model): class Maintainer(models.Model):
name = models.CharField(max_length=128) name = models.CharField(max_length=128)
email = models.CharField(max_length=128, unique=True) email = models.CharField(max_length=128, unique=True)
@ -16,6 +18,7 @@ class Maintainer(models.Model):
def __unicode__(self): def __unicode__(self):
return '%s <%s>' % (self.name, self.email) return '%s <%s>' % (self.name, self.email)
class Package(models.Model): class Package(models.Model):
category = models.CharField(max_length=128) category = models.CharField(max_length=128)
name = models.CharField(max_length=128) name = models.CharField(max_length=128)
@ -30,15 +33,18 @@ class Package(models.Model):
n_overlay = models.IntegerField(default=0) n_overlay = models.IntegerField(default=0)
' And we also pre-compute last versions ' ' And we also pre-compute last versions '
last_version_gentoo = models.ForeignKey('Version', blank=True, null=True, last_version_gentoo = models.ForeignKey(
related_name="last_version_gentoo", 'Version', blank=True, null=True, related_name="last_version_gentoo",
on_delete=models.SET_NULL) on_delete=models.SET_NULL
last_version_overlay = models.ForeignKey('Version', blank=True, null=True, )
related_name="last_version_overlay", last_version_overlay = models.ForeignKey(
on_delete=models.SET_NULL) 'Version', blank=True, null=True, related_name="last_version_overlay",
last_version_upstream = models.ForeignKey('Version', blank=True, null=True, on_delete=models.SET_NULL
related_name="last_version_upstream", )
on_delete=models.SET_NULL) last_version_upstream = models.ForeignKey(
'Version', blank=True, null=True, related_name="last_version_upstream",
on_delete=models.SET_NULL
)
def __unicode__(self): def __unicode__(self):
return '%s/%s' % (self.category, self.name) return '%s/%s' % (self.category, self.name)
@ -46,6 +52,7 @@ class Package(models.Model):
class Meta: class Meta:
unique_together = ['category', 'name'] unique_together = ['category', 'name']
class Version(models.Model): class Version(models.Model):
package = models.ForeignKey(Package) package = models.ForeignKey(Package)
slot = models.CharField(max_length=128) slot = models.CharField(max_length=128)
@ -57,13 +64,15 @@ class Version(models.Model):
alive = models.BooleanField(default=True, db_index=True) alive = models.BooleanField(default=True, db_index=True)
def __unicode__(self): def __unicode__(self):
return '%s/%s-%s-%s:%s [%s]' % (self.package.category, self.package.name, return '%s/%s-%s-%s:%s [%s]' % (
self.version, self.revision, self.slot, self.package.category, self.package.name, self.version,
self.overlay) self.revision, self.slot, self.overlay
)
class Meta: class Meta:
unique_together = ['package', 'slot', 'revision', 'version', 'overlay'] unique_together = ['package', 'slot', 'revision', 'version', 'overlay']
class VersionLog(models.Model): class VersionLog(models.Model):
VERSION_ADDED = 1 VERSION_ADDED = 1
VERSION_REMOVED = 2 VERSION_REMOVED = 2
@ -87,58 +96,67 @@ class VersionLog(models.Model):
def __unicode__(self): def __unicode__(self):
txt = '+ ' if self.action == self.VERSION_ADDED else '- ' txt = '+ ' if self.action == self.VERSION_ADDED else '- '
txt += '%s/%s-%s-%s:%s [%s]' % (self.package.category, self.package.name, txt += '%s/%s-%s-%s:%s [%s]' % (
self.version, self.revision, self.slot, self.package.category, self.package.name, self.version,
self.overlay if self.overlay else '<upstream>') self.revision, self.slot,
self.overlay if self.overlay else '<upstream>'
)
return txt return txt
class EuscanResult(models.Model): class EuscanResult(models.Model):
package = models.ForeignKey(Package) package = models.ForeignKey(Package)
datetime = models.DateTimeField() datetime = models.DateTimeField()
result = models.TextField(blank=True) result = models.TextField(blank=True)
# Keep data for charts # Keep data for charts
class Log(models.Model): class Log(models.Model):
datetime = models.DateTimeField() datetime = models.DateTimeField()
' Packages up to date in the main portage tree ' ' Packages up to date in the main portage tree '
n_packages_gentoo = models.IntegerField(default=0) n_packages_gentoo = models.IntegerField(default=0)
' Packages up to date in an overlay ' ' Packages up to date in an overlay '
n_packages_overlay = models.IntegerField(default=0) n_packages_overlay = models.IntegerField(default=0)
' Packages outdated ' ' Packages outdated '
n_packages_outdated = models.IntegerField(default=0) n_packages_outdated = models.IntegerField(default=0)
' Versions in the main portage tree ' ' Versions in the main portage tree '
n_versions_gentoo = models.IntegerField(default=0) n_versions_gentoo = models.IntegerField(default=0)
' Versions in overlays ' ' Versions in overlays '
n_versions_overlay = models.IntegerField(default=0) n_versions_overlay = models.IntegerField(default=0)
' Upstream versions, not in the main tree or overlays ' ' Upstream versions, not in the main tree or overlays '
n_versions_upstream = models.IntegerField(default=0) n_versions_upstream = models.IntegerField(default=0)
def __unicode__(self): def __unicode__(self):
return u'[%d:%d:%d] [%d:%d:%d]' % \ return u'[%d:%d:%d] [%d:%d:%d]' % (
(self.n_packages_gentoo, self.n_packages_overlay, self.n_packages_outdated, \ self.n_packages_gentoo, self.n_packages_overlay,
self.n_versions_gentoo, self.n_versions_overlay, self.n_versions_upstream) self.n_packages_outdated, self.n_versions_gentoo,
self.n_versions_overlay, self.n_versions_upstream
)
class WorldLog(Log): class WorldLog(Log):
def __unicode__(self): def __unicode__(self):
return u'world ' + Log.__unicode__(self) return u'world ' + Log.__unicode__(self)
class CategoryLog(Log): class CategoryLog(Log):
category = models.CharField(max_length=128) category = models.CharField(max_length=128)
def __unicode__(self): def __unicode__(self):
return u'%s %s' % (self.category, Log.__unicode__(self)) return u'%s %s' % (self.category, Log.__unicode__(self))
class HerdLog(Log): class HerdLog(Log):
herd = models.ForeignKey(Herd) herd = models.ForeignKey(Herd)
def __unicode__(self): def __unicode__(self):
return u'%s %s' % (self.herd, Log.__unicode__(self)) return u'%s %s' % (self.herd, Log.__unicode__(self))
class MaintainerLog(Log): class MaintainerLog(Log):
maintainer = models.ForeignKey(Maintainer) maintainer = models.ForeignKey(Maintainer)
def __unicode__(self): def __unicode__(self):
return u'%s %s' % (self.maintainer, Log.__unicode__(self)) return u'%s %s' % (self.maintainer, Log.__unicode__(self))

3
euscanwww/djeuscan/templatetags/div.py
View File

@ -2,7 +2,8 @@ from django import template
register = template.Library() register = template.Library()
def div(value, arg=None): def div(value, arg=None):
return float(value)/float(arg) return float(value) / float(arg)
register.filter('div', div) register.filter('div', div)

14
euscanwww/djeuscan/templatetags/math_.py
View File

@ -1,11 +1,14 @@
from django.template import Node, Library from django.template import Library
import math import math
register = Library() register = Library()
# taken from http://lybniz2.sourceforge.net/safeeval.html # taken from http://lybniz2.sourceforge.net/safeeval.html
# make a list of safe functions # make a list of safe functions
math_safe_list = ['acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'cosh', 'degrees', 'e', 'exp', 'fabs', 'floor', 'fmod', 'frexp', 'hypot', 'ldexp', 'log', 'log10', 'modf', 'pi', 'pow', 'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh'] math_safe_list = ['acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'cosh',
'degrees', 'e', 'exp', 'fabs', 'floor', 'fmod', 'frexp',
'hypot', 'ldexp', 'log', 'log10', 'modf', 'pi', 'pow',
'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh']
# use the list to filter the local namespace # use the list to filter the local namespace
math_safe_dict = dict([(k, getattr(math, k)) for k in math_safe_list]) math_safe_dict = dict([(k, getattr(math, k)) for k in math_safe_list])
@ -13,11 +16,13 @@ math_safe_dict = dict([(k, getattr(math, k)) for k in math_safe_list])
# add any needed builtins back in. # add any needed builtins back in.
math_safe_dict['abs'] = abs math_safe_dict['abs'] = abs
@register.filter('math') @register.filter('math')
def math_(lopr, expr): def math_(lopr, expr):
"""Evals a math expression and returns it's value. """Evals a math expression and returns it's value.
"$1" is a placeholder. Insert "$1" in the expression where the value is to be used. All math functions such as abs, sin, cos, floor are supported. "$1" is a placeholder. Insert "$1" in the expression where the value is
to be used. All math functions such as abs, sin, cos, floor are supported.
Example, Example,
a. You will be redirected in {{ seconds|math:"$1 / 60.0" }} minutes a. You will be redirected in {{ seconds|math:"$1 / 60.0" }} minutes
b. Square of {{ x }} is {{ x|math:"$1 * $1" }} b. Square of {{ x }} is {{ x|math:"$1 * $1" }}
@ -25,5 +30,6 @@ def math_(lopr, expr):
d. Given x = {{ x }}, (2 + x) * 6 = {{ x|math:"(2 + $1) * 6" }} d. Given x = {{ x }}, (2 + x) * 6 = {{ x|math:"(2 + $1) * 6" }}
""" """
if lopr: if lopr:
return eval(expr.replace('$1', str(lopr)), {"__builtins__": None}, math_safe_dict) return eval(expr.replace('$1', str(lopr)), {"__builtins__": None},
math_safe_dict)
return '' return ''

3
euscanwww/djeuscan/templatetags/mul.py
View File

@ -2,7 +2,8 @@ from django import template
register = template.Library() register = template.Library()
def mul(value, arg=None): def mul(value, arg=None):
return value*arg return value * arg
register.filter('mul', mul) register.filter('mul', mul)

3
euscanwww/djeuscan/templatetags/packages.py
View File

@ -2,16 +2,19 @@ from django import template
register = template.Library() register = template.Library()
@register.inclusion_tag('euscan/_packages.html', takes_context=True) @register.inclusion_tag('euscan/_packages.html', takes_context=True)
def packages(context, packages): def packages(context, packages):
context['packages'] = packages context['packages'] = packages
return context return context
@register.inclusion_tag('euscan/_package_cols.html', takes_context=True) @register.inclusion_tag('euscan/_package_cols.html', takes_context=True)
def package_cols(context, infos): def package_cols(context, infos):
context['infos'] = infos context['infos'] = infos
return context return context
@register.inclusion_tag('euscan/_package_bar.html', takes_context=True) @register.inclusion_tag('euscan/_package_bar.html', takes_context=True)
def package_bar(context, infos): def package_bar(context, infos):
context['infos'] = infos context['infos'] = infos

3
euscanwww/djeuscan/templatetags/sub.py
View File

@ -2,7 +2,8 @@ from django import template
register = template.Library() register = template.Library()
def sub(value, arg=None): def sub(value, arg=None):
return value-arg return value - arg
register.filter('sub', sub) register.filter('sub', sub)

7
euscanwww/djeuscan/templatetags/timedelta.py
View File

@ -6,6 +6,7 @@ from datetime import datetime
register = template.Library() register = template.Library()
def timedelta(value, arg=None): def timedelta(value, arg=None):
if not value: if not value:
return '' return ''
@ -16,8 +17,8 @@ def timedelta(value, arg=None):
if settings.USE_TZ: if settings.USE_TZ:
cmp = make_aware(cmp, get_default_timezone()) cmp = make_aware(cmp, get_default_timezone())
if value > cmp: if value > cmp:
return "in %s" % timesince(cmp,value) return "in %s" % timesince(cmp, value)
else: else:
return "%s ago" % timesince(value,cmp) return "%s ago" % timesince(value, cmp)
register.filter('timedelta',timedelta) register.filter('timedelta', timedelta)

2
euscanwww/djeuscan/tests.py
View File

@ -7,6 +7,7 @@ Replace these with more appropriate tests for your application.
from django.test import TestCase from django.test import TestCase
class SimpleTest(TestCase): class SimpleTest(TestCase):
def test_basic_addition(self): def test_basic_addition(self):
""" """
@ -20,4 +21,3 @@ Another way to test that 1 + 1 is equal to 2.
>>> 1 + 1 == 2 >>> 1 + 1 == 2
True True
"""} """}

23
euscanwww/djeuscan/urls.py
View File

@ -1,29 +1,36 @@
from django.conf.urls.defaults import * from django.conf.urls.defaults import url, patterns, include
from feeds import * from feeds import PackageFeed, CategoryFeed, HerdFeed, MaintainerFeed, \
GlobalFeed
package_patterns = patterns('djeuscan.views', package_patterns = patterns('djeuscan.views',
url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/feed/$', PackageFeed(), name='package_feed'), url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/feed/$',
PackageFeed(), name='package_feed'),
(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/$', 'package'), (r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/$', 'package'),
) )
categories_patterns = patterns('djeuscan.views', categories_patterns = patterns('djeuscan.views',
(r'^(?P<category>[\w+][\w+.-]*)/(view/)?$', 'category'), (r'^(?P<category>[\w+][\w+.-]*)/(view/)?$', 'category'),
url(r'^(?P<category>[\w+][\w+.-]*)/feed/$', CategoryFeed(), name='category_feed'), url(r'^(?P<category>[\w+][\w+.-]*)/feed/$', CategoryFeed(),
(r'^(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', 'chart_category'), name='category_feed'),
(r'^(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$',
'chart_category'),
(r'^$', 'categories'), (r'^$', 'categories'),
) )
herds_patterns = patterns('djeuscan.views', herds_patterns = patterns('djeuscan.views',
(r'^(?P<herd>[\@\{\}\w+.-]*)/(view/)?$', 'herd'), (r'^(?P<herd>[\@\{\}\w+.-]*)/(view/)?$', 'herd'),
url(r'^(?P<herd>[\@\{\}\w+.-]*)/feed/$', HerdFeed(), name='herd_feed'), url(r'^(?P<herd>[\@\{\}\w+.-]*)/feed/$', HerdFeed(), name='herd_feed'),
(r'^(?P<herd>[\@\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', 'chart_herd'), (r'^(?P<herd>[\@\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$',
'chart_herd'),
(r'^$', 'herds'), (r'^$', 'herds'),
) )
maintainers_patterns = patterns('djeuscan.views', maintainers_patterns = patterns('djeuscan.views',
(r'^(?P<maintainer_id>\d+)/(view/)?$', 'maintainer'), (r'^(?P<maintainer_id>\d+)/(view/)?$', 'maintainer'),
url(r'^(?P<maintainer_id>\d+)/feed/$', MaintainerFeed(), name='maintainer_feed'), url(r'^(?P<maintainer_id>\d+)/feed/$', MaintainerFeed(),
(r'^(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$', 'chart_maintainer'), name='maintainer_feed'),
(r'^(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$',
'chart_maintainer'),
(r'^$', 'maintainers'), (r'^$', 'maintainers'),
) )

117
euscanwww/djeuscan/views.py
View File

@ -1,5 +1,5 @@
from annoying.decorators import render_to from annoying.decorators import render_to
from django.http import HttpResponse, Http404 from django.http import Http404
from django.shortcuts import get_object_or_404 from django.shortcuts import get_object_or_404
from django.db.models import Sum, Max from django.db.models import Sum, Max
@ -10,84 +10,117 @@ import charts
""" Views """ """ Views """
@render_to('euscan/index.html') @render_to('euscan/index.html')
def index(request): def index(request):
ctx = {} ctx = {}
ctx['n_packaged'] = charts.xint(Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum']) ctx['n_packaged'] = charts.xint(
ctx['n_overlay'] = charts.xint(Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum']) Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum']
ctx['n_versions'] = charts.xint(Package.objects.aggregate(Sum('n_versions'))['n_versions__sum']) )
ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - ctx['n_overlay'] ctx['n_overlay'] = charts.xint(
Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum']
)
ctx['n_versions'] = charts.xint(
Package.objects.aggregate(Sum('n_versions'))['n_versions__sum']
)
ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - \
ctx['n_overlay']
ctx['n_packages'] = Package.objects.count() ctx['n_packages'] = Package.objects.count()
ctx['n_herds'] = Herd.objects.count() ctx['n_herds'] = Herd.objects.count()
ctx['n_maintainers'] = Maintainer.objects.count() ctx['n_maintainers'] = Maintainer.objects.count()
try: try:
ctx['last_scan'] = EuscanResult.objects.get(id=EuscanResult.objects.aggregate(Max('id'))['id__max']).datetime ctx['last_scan'] = EuscanResult.objects.get(
id=EuscanResult.objects.aggregate(Max('id'))['id__max']
).datetime
except EuscanResult.DoesNotExist: except EuscanResult.DoesNotExist:
ctx['last_scan'] = None ctx['last_scan'] = None
return ctx return ctx
@render_to('euscan/logs.html') @render_to('euscan/logs.html')
def logs(request): def logs(request):
return {} return {}
@render_to('euscan/categories.html') @render_to('euscan/categories.html')
def categories(request): def categories(request):
categories = Package.objects.values('category').annotate(n_packaged=Sum('n_packaged'), categories = Package.objects.values('category').annotate(
n_overlay=Sum('n_overlay'), n_packaged=Sum('n_packaged'),
n_versions=Sum('n_versions')) n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')
)
return {'categories': categories}
return { 'categories' : categories }
@render_to('euscan/category.html') @render_to('euscan/category.html')
def category(request, category): def category(request, category):
packages = Package.objects.filter(category=category) packages = Package.objects.filter(category=category)
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream') packages = packages.select_related(
'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
)
print dir(packages[0]) print dir(packages[0])
if not packages: if not packages:
raise Http404 raise Http404
return { 'category' : category, 'packages' : packages } return {'category': category, 'packages': packages}
@render_to('euscan/herds.html') @render_to('euscan/herds.html')
def herds(request): def herds(request):
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of 'INNER JOIN' # FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of
# 'INNER JOIN'
herds = Package.objects.filter(herds__isnull=False) herds = Package.objects.filter(herds__isnull=False)
herds = herds.values('herds__herd').annotate(n_packaged=Sum('n_packaged'), herds = herds.values('herds__herd').annotate(
n_overlay=Sum('n_overlay'), n_packaged=Sum('n_packaged'),
n_versions=Sum('n_versions')) n_overlay=Sum('n_overlay'),
return { 'herds' : herds } n_versions=Sum('n_versions'))
return {'herds': herds}
@render_to('euscan/herd.html') @render_to('euscan/herd.html')
def herd(request, herd): def herd(request, herd):
herd = get_object_or_404(Herd, herd=herd) herd = get_object_or_404(Herd, herd=herd)
packages = Package.objects.filter(herds__id=herd.id) packages = Package.objects.filter(herds__id=herd.id)
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream') packages = packages.select_related(
return { 'herd' : herd, 'packages' : packages } 'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
)
return {'herd': herd, 'packages': packages}
@render_to('euscan/maintainers.html') @render_to('euscan/maintainers.html')
def maintainers(request): def maintainers(request):
maintainers = Package.objects.filter(maintainers__isnull=False) maintainers = Package.objects.filter(maintainers__isnull=False)
maintainers = maintainers.values('maintainers__id', 'maintainers__name', 'maintainers__email') maintainers = maintainers.values(
maintainers = maintainers.annotate(n_packaged=Sum('n_packaged'), 'maintainers__id', 'maintainers__name', 'maintainers__email'
n_overlay=Sum('n_overlay'), )
n_versions=Sum('n_versions')) maintainers = maintainers.annotate(
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')
)
return {'maintainers': maintainers}
return { 'maintainers' : maintainers }
@render_to('euscan/maintainer.html') @render_to('euscan/maintainer.html')
def maintainer(request, maintainer_id): def maintainer(request, maintainer_id):
maintainer = get_object_or_404(Maintainer, id=maintainer_id) maintainer = get_object_or_404(Maintainer, id=maintainer_id)
packages = Package.objects.filter(maintainers__id=maintainer.id) packages = Package.objects.filter(maintainers__id=maintainer.id)
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream') packages = packages.select_related(
return { 'maintainer' : maintainer, 'packages' : packages } 'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
)
return {'maintainer': maintainer, 'packages': packages}
@render_to('euscan/overlays.html') @render_to('euscan/overlays.html')
def overlays(request): def overlays(request):
overlays = Package.objects.values('version__overlay') overlays = Package.objects.values('version__overlay')
overlays = overlays.exclude(version__overlay='') overlays = overlays.exclude(version__overlay='')
overlays = overlays.distinct() overlays = overlays.distinct()
return { 'overlays' : overlays } return {'overlays': overlays}
@render_to('euscan/overlay.html') @render_to('euscan/overlay.html')
def overlay(request, overlay): def overlay(request, overlay):
@ -97,7 +130,8 @@ def overlay(request, overlay):
packages = packages.filter(version__overlay=overlay).distinct() packages = packages.filter(version__overlay=overlay).distinct()
if not packages: if not packages:
raise Http404 raise Http404
return { 'overlay' : overlay, 'packages' : packages } return {'overlay': overlay, 'packages': packages}
@render_to('euscan/package.html') @render_to('euscan/package.html')
def package(request, category, package): def package(request, category, package):
@ -120,19 +154,23 @@ def package(request, category, package):
packaged = sorted(packaged, key=version_key) packaged = sorted(packaged, key=version_key)
upstream = sorted(upstream, key=version_key) upstream = sorted(upstream, key=version_key)
log = EuscanResult.objects.filter(package=package).order_by('-datetime')[:1] log = EuscanResult.objects.filter(package=package).\
order_by('-datetime')[:1]
log = log[0] if log else None log = log[0] if log else None
vlog = VersionLog.objects.filter(package=package).order_by('-id') vlog = VersionLog.objects.filter(package=package).order_by('-id')
return { 'package' : package, 'packaged' : packaged,
'upstream' : upstream, 'log' : log, 'vlog' : vlog } return {'package': package, 'packaged': packaged,
'upstream': upstream, 'log': log, 'vlog': vlog}
@render_to('euscan/world.html') @render_to('euscan/world.html')
def world(request): def world(request):
world_form = WorldForm() world_form = WorldForm()
packages_form = PackagesForm() packages_form = PackagesForm()
return { 'world_form' : world_form, return {'world_form': world_form,
'packages_form' : packages_form } 'packages_form': packages_form}
@render_to('euscan/world_scan.html') @render_to('euscan/world_scan.html')
def world_scan(request): def world_scan(request):
@ -157,28 +195,34 @@ def world_scan(request):
except: except:
pass pass
return { 'packages' : packages } return {'packages': packages}
@render_to("euscan/about.html") @render_to("euscan/about.html")
def about(request): def about(request):
return {} return {}
@render_to("euscan/api.html") @render_to("euscan/api.html")
def api(request): def api(request):
return {} return {}
@render_to("euscan/statistics.html") @render_to("euscan/statistics.html")
def statistics(request): def statistics(request):
return {} return {}
def chart(request, **kwargs): def chart(request, **kwargs):
from django.views.static import serve from django.views.static import serve
chart = kwargs['chart'] if 'chart' in kwargs else None chart = kwargs['chart'] if 'chart' in kwargs else None
if 'maintainer_id' in kwargs: if 'maintainer_id' in kwargs:
kwargs['maintainer'] = get_object_or_404(Maintainer, id=kwargs['maintainer_id']) kwargs['maintainer'] = get_object_or_404(
Maintainer,
id=kwargs['maintainer_id']
)
if 'herd' in kwargs: if 'herd' in kwargs:
kwargs['herd'] = get_object_or_404(Herd, herd=kwargs['herd']) kwargs['herd'] = get_object_or_404(Herd, herd=kwargs['herd'])
@ -202,11 +246,14 @@ def chart(request, **kwargs):
return serve(request, path, document_root=charts.CHARTS_ROOT) return serve(request, path, document_root=charts.CHARTS_ROOT)
def chart_maintainer(request, **kwargs): def chart_maintainer(request, **kwargs):
return chart(request, **kwargs) return chart(request, **kwargs)
def chart_herd(request, **kwargs): def chart_herd(request, **kwargs):
return chart(request, **kwargs) return chart(request, **kwargs)
def chart_category(request, **kwargs): def chart_category(request, **kwargs):
return chart(request, **kwargs) return chart(request, **kwargs)

1
euscanwww/euscanwww/urls.py
View File

@ -22,4 +22,3 @@ if settings.DEBUG:
'document_root': os.path.join(settings.EUSCAN_ROOT, 'htdocs'), 'document_root': os.path.join(settings.EUSCAN_ROOT, 'htdocs'),
}), }),
) )

3
euscanwww/manage.py
View File

@ -1,5 +1,6 @@
#!/usr/bin/env python #!/usr/bin/env python
import os, sys import os
import sys
if __name__ == "__main__": if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "euscanwww.settings") os.environ.setdefault("DJANGO_SETTINGS_MODULE", "euscanwww.settings")

5
euscanwww/scripts/regen-seq.py
View File

@ -1,4 +1,5 @@
import os, sys import os
import sys
sys.path.insert(0, '../') sys.path.insert(0, '../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'euscanwww.settings' os.environ['DJANGO_SETTINGS_MODULE'] = 'euscanwww.settings'
@ -19,6 +20,6 @@ try:
rel_name = seq_name.split("_id_seq")[0] rel_name = seq_name.split("_id_seq")[0]
to_update.append((seq_name, rel_name,)) to_update.append((seq_name, rel_name,))
for row in to_update: for row in to_update:
c.execute(r"SELECT setval('%s', max(id)) FROM %s"%row) c.execute(r"SELECT setval('%s', max(id)) FROM %s" % row)
finally: finally:
c.close() c.close()

70
euscanwww/templates/_base.html
View File

@ -15,7 +15,7 @@
<body> <body>
<div id="header"> <div id="header">
<a href="http://www.gentoo.org"> <a href="http://www.gentoo.org">
<img id="logo" src="{{ STATIC_URL }}img/gentoo_org.png" /> <img id="logo" src="{{ STATIC_URL }}img/gentoo_org.png" />
</a> </a>
{% block header %}<h1>Ebuild Upstream Scanner (euscan)</h1>{% endblock %} {% block header %}<h1>Ebuild Upstream Scanner (euscan)</h1>{% endblock %}
</div> </div>
@ -24,44 +24,44 @@
</div> </div>
<div id="menus"> <div id="menus">
{% block menus %} {% block menus %}
<div class="menu"> <div class="menu">
<ul> <ul>
{% block menu %} {% block menu %}
<li><a href="{% url djeuscan.views.index %}">Home</a></li> <li><a href="{% url djeuscan.views.index %}">Home</a></li>
<li><a href="{% url djeuscan.views.categories %}">Categories</a></li> <li><a href="{% url djeuscan.views.categories %}">Categories</a></li>
<li><a href="{% url djeuscan.views.herds %}">Herds</a></li> <li><a href="{% url djeuscan.views.herds %}">Herds</a></li>
<li><a href="{% url djeuscan.views.maintainers %}">Maintainers</a></li> <li><a href="{% url djeuscan.views.maintainers %}">Maintainers</a></li>
<li><a href="{% url djeuscan.views.overlays %}">Overlays</a></li> <li><a href="{% url djeuscan.views.overlays %}">Overlays</a></li>
<li><a href="{% url djeuscan.views.world %}">Scan World</a></li> <li><a href="{% url djeuscan.views.world %}">Scan World</a></li>
<li><a href="{% url djeuscan.views.statistics %}">Statistics</a></li> <li><a href="{% url djeuscan.views.statistics %}">Statistics</a></li>
<!-- <!--
<li>---</li> <li>---</li>
<li><a href="#">Login</a></li> <li><a href="#">Login</a></li>
<li><a href="#">Register</a></li> <li><a href="#">Register</a></li>
--> -->
<li>---</li> <li>---</li>
{% block menu_feed %} {% block menu_feed %}
<li> <li>
<img src="{{ STATIC_URL }}img/feed.png" alt="feed" /> <img src="{{ STATIC_URL }}img/feed.png" alt="feed" />
<a title="Global Feed" href="{% url global_feed %}">Global Feed</a> <a title="Global Feed" href="{% url global_feed %}">Global Feed</a>
</li> </li>
{% endblock %} {% endblock %}
<li>---</li> <li>---</li>
<li><a href="{% url djeuscan.views.api %}">API</a></li> <li><a href="{% url djeuscan.views.api %}">API</a></li>
<li><a href="{% url djeuscan.views.about %}">About</a></li> <li><a href="{% url djeuscan.views.about %}">About</a></li>
{% endblock %} {% endblock %}
</ul> </ul>
</div> </div>
{% endblock %} {% endblock %}
</div> </div>
<div id="footer"> <div id="footer">
<p> <p>
Questions, Comments, Corrections ? Questions, Comments, Corrections ?
Email: corentin.chary at gmail.com<br /> Email: corentin.chary at gmail.com<br />
Copyright (C) 2011 <strong>Corentin Chary</strong><br /> Copyright (C) 2011 <strong>Corentin Chary</strong><br />
Original Gentoo artwork and logos copyright (C) Gentoo Foundation.<br /> Original Gentoo artwork and logos copyright (C) Gentoo Foundation.<br />
Design inspired by (stolen from) gentoo.org and bugs.gentoo.org.<br /> Design inspired by (stolen from) gentoo.org and bugs.gentoo.org.<br />
<em>This site is not an official Gentoo website.</em> <em>This site is not an official Gentoo website.</em>
</p> </p>
</div> </div>
</body> </body>

2
euscanwww/templates/euscan/_packages.html
View File

@ -19,7 +19,7 @@
<tr> <tr>
<td> <td>
<a href="{% url djeuscan.views.package package.category package.name %}"> <a href="{% url djeuscan.views.package package.category package.name %}">
{{ package.category }}/{{ package.name }} {{ package.category }}/{{ package.name }}
</a> </a>
{% package_bar package %} {% package_bar package %}
</td> </td>

2
euscanwww/templates/euscan/category.html
View File

@ -47,7 +47,7 @@
<dd><img src="{% url djeuscan.views.chart_category category 'versions-monthly-small' %}" /></dd> <dd><img src="{% url djeuscan.views.chart_category category 'versions-monthly-small' %}" /></dd>
<dt>Packages</dt> <dt>Packages</dt>
<dd><img src="{% url djeuscan.views.chart_category category 'packages-monthly-small' %}" /></dd> <dd><img src="{% url djeuscan.views.chart_category category 'packages-monthly-small' %}" /></dd>
</dl> </dl>
</div> </div>
{% endblock %} {% endblock %}

2
euscanwww/templates/euscan/herd.html
View File

@ -45,7 +45,7 @@
<dd><img src="{% url djeuscan.views.chart_herd herd.herd 'versions-monthly-small' %}" /></dd> <dd><img src="{% url djeuscan.views.chart_herd herd.herd 'versions-monthly-small' %}" /></dd>
<dt>Packages</dt> <dt>Packages</dt>
<dd><img src="{% url djeuscan.views.chart_herd herd.herd 'packages-monthly-small' %}" /></dd> <dd><img src="{% url djeuscan.views.chart_herd herd.herd 'packages-monthly-small' %}" /></dd>
</dl> </dl>
</div> </div>
{% endblock %} {% endblock %}

2
euscanwww/templates/euscan/maintainer.html
View File

@ -45,7 +45,7 @@
<dd><img src="{% url djeuscan.views.chart_maintainer maintainer.id 'versions-monthly-small' %}" /></dd> <dd><img src="{% url djeuscan.views.chart_maintainer maintainer.id 'versions-monthly-small' %}" /></dd>
<dt>Packages</dt> <dt>Packages</dt>
<dd><img src="{% url djeuscan.views.chart_maintainer maintainer.id 'packages-monthly-small' %}" /></dd> <dd><img src="{% url djeuscan.views.chart_maintainer maintainer.id 'packages-monthly-small' %}" /></dd>
</dl> </dl>
</div> </div>
{% endblock %} {% endblock %}

10
euscanwww/templates/euscan/maintainers.html
View File

@ -24,11 +24,11 @@
<tr> <tr>
<td> <td>
<a href="{% url djeuscan.views.maintainer maintainer.maintainers__id %}"> <a href="{% url djeuscan.views.maintainer maintainer.maintainers__id %}">
{% if maintainer.maintainers__name != maintainer.maintainers__email %} {% if maintainer.maintainers__name != maintainer.maintainers__email %}
{{ maintainer.maintainers__name }} &lt;{{ maintainer.maintainers__email }}&gt; {{ maintainer.maintainers__name }} &lt;{{ maintainer.maintainers__email }}&gt;
{% else %} {% else %}
{{ maintainer.maintainers__name }} {{ maintainer.maintainers__name }}
{% endif %} {% endif %}
</a> </a>
{% package_bar maintainer %} {% package_bar maintainer %}
</td> </td>

36
euscanwww/templates/euscan/package.html
View File

@ -51,7 +51,7 @@
{% for maintainer in package.maintainers.all %} {% for maintainer in package.maintainers.all %}
{% if maintainer.name != maintainer.email %} {% if maintainer.name != maintainer.email %}
<a href="{% url djeuscan.views.maintainer maintainer.id %}"> <a href="{% url djeuscan.views.maintainer maintainer.id %}">
{{ maintainer.name }} {{ maintainer.name }}
</a> </a>
&lt;{{ maintainer.email }}&gt; &lt;{{ maintainer.email }}&gt;
{% else %} {% else %}
@ -68,12 +68,12 @@
<ul> <ul>
{% for version in packaged %} {% for version in packaged %}
<li id="{{ version.version }}-{{version.revision }}:{{ version.slot }}-[{{ version.overlay }}]"> <li id="{{ version.version }}-{{version.revision }}:{{ version.slot }}-[{{ version.overlay }}]">
{% if version.overlay == "gentoo" %} {% if version.overlay == "gentoo" %}
<img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" /> <img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
{% else %} {% else %}
<img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" /> <img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
{% endif %} {% endif %}
{{ version.version }}-{{ version.revision }} :{{ version.slot }} [{{ version.overlay }}] {{ version.version }}-{{ version.revision }} :{{ version.slot }} [{{ version.overlay }}]
</li> </li>
{% endfor %} {% endfor %}
</ul> </ul>
@ -85,8 +85,8 @@
<ul> <ul>
{% for version in upstream %} {% for version in upstream %}
<li> <li>
<img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" /> <img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
{{ version.version }} - {{ version.urls }} {{ version.version }} - {{ version.urls }}
</li> </li>
{% endfor %} {% endfor %}
</ul> </ul>
@ -101,14 +101,16 @@
{% else %} {% else %}
<li class="removed"> <li class="removed">
{% endif %} {% endif %}
{% if version.overlay == "gentoo" %} {% if version.overlay == "gentoo" %}
<img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" /> <img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
{% else %}{% if version.overlay %} {% elif version.overlay %}
<img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" /> <img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
{% else %} {% else %}
<img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" /> <img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
{% endif %}{% endif %} {% endif %}
{{ version }} - {{ version.datetime }}
{{ version }} - {{ version.datetime }}
</li> </li>
{% endfor %} {% endfor %}
</ul> </ul>

31
pym/euscan/__init__.py
View File

@ -5,7 +5,6 @@
__version__ = "git" __version__ = "git"
import sys
from portage.output import EOutput from portage.output import EOutput
@ -19,15 +18,16 @@ CONFIG = {
'brute-force-false-watermark': 50, 'brute-force-false-watermark': 50,
'scan-dir': True, 'scan-dir': True,
'oneshot': True, 'oneshot': True,
'user-agent' : 'escan (http://euscan.iksaif.net)', 'user-agent': 'escan (http://euscan.iksaif.net)',
'skip-robots-txt' : False, 'skip-robots-txt': False,
'cache' : False 'cache': False
} }
output = EOutput(CONFIG['quiet']) output = EOutput(CONFIG['quiet'])
BLACKLIST_VERSIONS = [ BLACKLIST_VERSIONS = [
# Compatibility package for running binaries linked against a pre gcc 3.4 libstdc++, won't be updated # Compatibility package for running binaries linked against a
# pre gcc 3.4 libstdc++, won't be updated
'>=sys-libs/libstdc++-v3-3.4', '>=sys-libs/libstdc++-v3-3.4',
] ]
@ -39,21 +39,24 @@ BLACKLIST_PACKAGES = [
] ]
SCANDIR_BLACKLIST_URLS = [ SCANDIR_BLACKLIST_URLS = [
'mirror://rubygems/(.*)', # Not browsable 'mirror://rubygems/(.*)', # Not browsable
'mirror://gentoo/(.*)' # Directory too big 'mirror://gentoo/(.*)' # Directory too big
] ]
BRUTEFORCE_BLACKLIST_PACKAGES = [ BRUTEFORCE_BLACKLIST_PACKAGES = [
'net-zope/plonepopoll' # infinite loop any http://plone.org/products/plonepopoll/releases/*/plonepopoll-2-6-1.tgz link will work # infinite loop any
# http://plone.org/products/plonepopoll/releases/*/plonepopoll-2-6-1.tgz
# link will work
'net-zope/plonepopoll'
] ]
BRUTEFORCE_BLACKLIST_URLS = [ BRUTEFORCE_BLACKLIST_URLS = [
'http://(.*)dockapps.org/download.php/id/(.*)', # infinite loop 'http://(.*)dockapps.org/download.php/id/(.*)', # infinite loop
'http://hydra.nixos.org/build/(.*)', # infinite loop 'http://hydra.nixos.org/build/(.*)', # infinite loop
'http://www.rennings.net/gentoo/distfiles/(.*)', # Doesn't respect 404, infinite loop 'http://www.rennings.net/gentoo/distfiles/(.*)', # Doesn't respect 404, infinite loop
'http://art.gnome.org/download/(.*)', # Doesn't respect 404, infinite loop 'http://art.gnome.org/download/(.*)', # Doesn't respect 404, infinite loop
'http://barelysufficient.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop 'http://barelysufficient.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop
'http://olemarkus.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop 'http://olemarkus.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop
] ]
ROBOTS_TXT_BLACKLIST_DOMAINS = [ ROBOTS_TXT_BLACKLIST_DOMAINS = [

5
pym/euscan/handlers/__init__.py
View File

@ -1,6 +1,7 @@
from euscan.handlers import generic, php, pypi, rubygem, kde, cpan from euscan.handlers import generic, php, pypi, rubygem, kde, cpan
handlers = [ kde, php, pypi, rubygem, cpan, generic ] handlers = [kde, php, pypi, rubygem, cpan, generic]
def find_best_handler(cpv, url): def find_best_handler(cpv, url):
for handler in handlers: for handler in handlers:
@ -8,12 +9,14 @@ def find_best_handler(cpv, url):
return handler return handler
return None return None
def scan(cpv, url): def scan(cpv, url):
handler = find_best_handler(cpv, url) handler = find_best_handler(cpv, url)
if handler: if handler:
return handler.scan(cpv, url) return handler.scan(cpv, url)
return [] return []
def brute_force(cpv, url): def brute_force(cpv, url):
handler = find_best_handler(cpv, url) handler = find_best_handler(cpv, url)
if handler: if handler:

21
pym/euscan/handlers/cpan.py
View File

@ -8,9 +8,11 @@ import euscan
_cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*") _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*")
def can_handle(cpv, url): def can_handle(cpv, url):
return url.startswith('mirror://cpan/') return url.startswith('mirror://cpan/')
def guess_package(cp, url): def guess_package(cp, url):
match = _cpan_package_name_re.search(url) match = _cpan_package_name_re.search(url)
@ -27,6 +29,7 @@ def guess_package(cp, url):
return pkg return pkg
def gentoo_mangle_version(up_pv): def gentoo_mangle_version(up_pv):
pv = "" pv = ""
@ -39,17 +42,20 @@ def gentoo_mangle_version(up_pv):
c = up_pv[i] c = up_pv[i]
pv += c pv += c
digits += int(c.isdigit()) digits += int(c.isdigit())
if c == '.': digits = 0 if c == '.':
digits = 0
else: else:
pv = up_pv pv = up_pv
return helpers.gentoo_mangle_version(pv) return helpers.gentoo_mangle_version(pv)
def cpan_trim_version(pv): def cpan_trim_version(pv):
pv = re.sub('^[a-zA-Z]+', '', pv) pv = re.sub('^[a-zA-Z]+', '', pv)
pv = re.sub('[a-zA-Z]$', '', pv) pv = re.sub('[a-zA-Z]$', '', pv)
return pv return pv
def cpan_mangle_version(pv): def cpan_mangle_version(pv):
pos = pv.find('.') pos = pv.find('.')
if pos < 0: if pos < 0:
@ -59,6 +65,7 @@ def cpan_mangle_version(pv):
up_pv = cpan_trim_version(up_pv) up_pv = cpan_trim_version(up_pv)
return up_pv return up_pv
def cpan_vercmp(cp, a, b): def cpan_vercmp(cp, a, b):
try: try:
return float(a) - float(b) return float(a) - float(b)
@ -68,6 +75,7 @@ def cpan_vercmp(cp, a, b):
else: else:
return 1 return 1
def scan(cpv, url): def scan(cpv, url):
cp, ver, rev = portage.pkgsplit(cpv) cp, ver, rev = portage.pkgsplit(cpv)
pkg = guess_package(cp, url) pkg = guess_package(cp, url)
@ -107,15 +115,20 @@ def scan(cpv, url):
if helpers.version_filtered(cp, up_ver, up_pv, cpan_vercmp): if helpers.version_filtered(cp, up_ver, up_pv, cpan_vercmp):
continue continue
url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % \ url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % (
(version['cpanid'][0], version['cpanid'][0:1], version['cpanid'], version['archive']) version['cpanid'][0],
version['cpanid'][0:1],
version['cpanid'],
version['archive']
)
if url == orig_url: if url == orig_url:
continue continue
ret.append(( url, pv )) ret.append((url, pv))
return ret return ret
def brute_force(cpv, url): def brute_force(cpv, url):
return [] return []

31
pym/euscan/handlers/generic.py
View File

@ -6,10 +6,12 @@ from BeautifulSoup import BeautifulSoup
import portage import portage
from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \
BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS
from euscan import helpers from euscan import helpers
import euscan import euscan
def scan_html(data, url, pattern): def scan_html(data, url, pattern):
soup = BeautifulSoup(data) soup = BeautifulSoup(data)
results = [] results = []
@ -28,6 +30,7 @@ def scan_html(data, url, pattern):
return results return results
def scan_ftp(data, url, pattern): def scan_ftp(data, url, pattern):
buf = StringIO.StringIO(data) buf = StringIO.StringIO(data)
results = [] results = []
@ -40,6 +43,7 @@ def scan_ftp(data, url, pattern):
return results return results
def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
if not steps: if not steps:
return [] return []
@ -91,6 +95,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
return versions return versions
def scan(cpv, url): def scan(cpv, url):
for bu in SCANDIR_BLACKLIST_URLS: for bu in SCANDIR_BLACKLIST_URLS:
if re.match(bu, url): if re.match(bu, url):
@ -107,13 +112,17 @@ def scan(cpv, url):
if ver not in resolved_url: if ver not in resolved_url:
newver = helpers.version_change_end_sep(ver) newver = helpers.version_change_end_sep(ver)
if newver and newver in resolved_url: if newver and newver in resolved_url:
euscan.output.einfo("Version: using %s instead of %s" % (newver, ver)) euscan.output.einfo(
"Version: using %s instead of %s" % (newver, ver)
)
ver = newver ver = newver
template = helpers.template_from_url(resolved_url, ver) template = helpers.template_from_url(resolved_url, ver)
if '${' not in template: if '${' not in template:
euscan.output.einfo("Url doesn't seems to depend on version: %s not found in %s" euscan.output.einfo(
% (ver, resolved_url)) "Url doesn't seems to depend on version: %s not found in %s" %
(ver, resolved_url)
)
return [] return []
else: else:
euscan.output.einfo("Scanning: %s" % template) euscan.output.einfo("Scanning: %s" % template)
@ -121,6 +130,7 @@ def scan(cpv, url):
steps = helpers.generate_scan_paths(template) steps = helpers.generate_scan_paths(template)
return scan_directory_recursive(cp, ver, rev, "", steps, url) return scan_directory_recursive(cp, ver, rev, "", steps, url)
def brute_force(cpv, url): def brute_force(cpv, url):
cp, ver, rev = portage.pkgsplit(cpv) cp, ver, rev = portage.pkgsplit(cpv)
@ -155,8 +165,9 @@ def brute_force(cpv, url):
template = helpers.template_from_url(url, ver) template = helpers.template_from_url(url, ver)
if '${PV}' not in template: if '${PV}' not in template:
euscan.output.einfo("Url doesn't seems to depend on full version: %s not found in %s" euscan.output.einfo(
% (ver, url)) "Url doesn't seems to depend on full version: %s not found in %s" %
(ver, url))
return [] return []
else: else:
euscan.output.einfo("Brute forcing: %s" % template) euscan.output.einfo("Brute forcing: %s" % template)
@ -187,11 +198,14 @@ def brute_force(cpv, url):
result.append([url, version]) result.append([url, version])
if len(result) > CONFIG['brute-force-false-watermark']: if len(result) > CONFIG['brute-force-false-watermark']:
euscan.output.einfo("Broken server detected ! Skipping brute force.") euscan.output.einfo(
"Broken server detected ! Skipping brute force."
)
return [] return []
if CONFIG["brute-force-recursive"]: if CONFIG["brute-force-recursive"]:
for v in helpers.gen_versions(list(components), CONFIG["brute-force"]): for v in helpers.gen_versions(list(components),
CONFIG["brute-force"]):
if v not in versions and tuple(v) not in done: if v not in versions and tuple(v) not in done:
versions.append(v) versions.append(v)
@ -200,5 +214,6 @@ def brute_force(cpv, url):
return result return result
def can_handle(cpv, url): def can_handle(cpv, url):
return True return True

4
pym/euscan/handlers/kde.py
View File

@ -1,10 +1,12 @@
from euscan.handlers import generic from euscan.handlers import generic
def can_handle(cpv, url): def can_handle(cpv, url):
if url.startswith('mirror://kde/'): if url.startswith('mirror://kde/'):
return True return True
return False return False
def clean_results(results): def clean_results(results):
ret = [] ret = []
@ -15,6 +17,7 @@ def clean_results(results):
return ret return ret
def scan(cpv, url): def scan(cpv, url):
results = generic.scan(cpv, url) results = generic.scan(cpv, url)
@ -24,6 +27,7 @@ def scan(cpv, url):
return clean_results(results) return clean_results(results)
def brute_force(cpv, url): def brute_force(cpv, url):
results = generic.brute_force(cpv, url) results = generic.brute_force(cpv, url)

6
pym/euscan/handlers/php.py
View File

@ -6,6 +6,7 @@ import xml.dom.minidom
from euscan import helpers from euscan import helpers
import euscan import euscan
def can_handle(cpv, url): def can_handle(cpv, url):
if url.startswith('http://pear.php.net/get/'): if url.startswith('http://pear.php.net/get/'):
return True return True
@ -13,6 +14,7 @@ def can_handle(cpv, url):
return True return True
return False return False
def guess_package_and_channel(cp, url): def guess_package_and_channel(cp, url):
match = re.search('http://(.*)/get/(.*)-(.*).tgz', url) match = re.search('http://(.*)/get/(.*)-(.*).tgz', url)
@ -24,6 +26,7 @@ def guess_package_and_channel(cp, url):
return pkg, host return pkg, host
def scan(cpv, url): def scan(cpv, url):
cp, ver, rev = portage.pkgsplit(cpv) cp, ver, rev = portage.pkgsplit(cpv)
pkg, channel = guess_package_and_channel(cp, url) pkg, channel = guess_package_and_channel(cp, url)
@ -61,9 +64,10 @@ def scan(cpv, url):
if url == orig_url: if url == orig_url:
continue continue
ret.append(( url, pv )) ret.append((url, pv))
return ret return ret
def brute_force(cpv, url): def brute_force(cpv, url):
return [] return []

10
pym/euscan/handlers/pypi.py
View File

@ -1,5 +1,4 @@
import xmlrpclib import xmlrpclib
import pprint
import re import re
import portage import portage
@ -7,9 +6,11 @@ import portage
from euscan import helpers from euscan import helpers
import euscan import euscan
def can_handle(cpv, url): def can_handle(cpv, url):
return url.startswith('mirror://pypi/') return url.startswith('mirror://pypi/')
def guess_package(cp, url): def guess_package(cp, url):
match = re.search('mirror://pypi/\w+/(.*)/.*', url) match = re.search('mirror://pypi/\w+/(.*)/.*', url)
if match: if match:
@ -19,10 +20,10 @@ def guess_package(cp, url):
return pkg return pkg
def scan(cpv, url): def scan(cpv, url):
'http://wiki.python.org/moin/PyPiXmlRpc' 'http://wiki.python.org/moin/PyPiXmlRpc'
package = guess_package(cpv, url) package = guess_package(cpv, url)
euscan.output.einfo("Using PyPi XMLRPC: " + package) euscan.output.einfo("Using PyPi XMLRPC: " + package)
@ -44,10 +45,11 @@ def scan(cpv, url):
if helpers.version_filtered(cp, ver, pv): if helpers.version_filtered(cp, ver, pv):
continue continue
urls = client.release_urls(package, up_pv) urls = client.release_urls(package, up_pv)
urls = " ".join([ infos['url'] for infos in urls ]) urls = " ".join([infos['url'] for infos in urls])
ret.append(( urls, pv )) ret.append((urls, pv))
return ret return ret
def brute_force(cpv, url): def brute_force(cpv, url):
return [] return []

9
pym/euscan/handlers/rubygem.py
View File

@ -6,9 +6,11 @@ import urllib2
from euscan import helpers from euscan import helpers
import euscan import euscan
def can_handle(cpv, url): def can_handle(cpv, url):
return url.startswith('mirror://rubygems/') return url.startswith('mirror://rubygems/')
def guess_gem(cpv, url): def guess_gem(cpv, url):
match = re.search('mirror://rubygems/(.*).gem', url) match = re.search('mirror://rubygems/(.*).gem', url)
if match: if match:
@ -23,12 +25,14 @@ def guess_gem(cpv, url):
return pkg return pkg
def scan(cpv, url): def scan(cpv, url):
'http://guides.rubygems.org/rubygems-org-api/#gemversion' 'http://guides.rubygems.org/rubygems-org-api/#gemversion'
gem = guess_gem(cpv, url) gem = guess_gem(cpv, url)
if not gem: if not gem:
euscan.output.eerror("Can't guess gem name using %s and %s" % (cpv, url)) euscan.output.eerror("Can't guess gem name using %s and %s" % \
(cpv, url))
return [] return []
url = 'http://rubygems.org/api/v1/versions/%s.json' % gem url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
@ -61,9 +65,10 @@ def scan(cpv, url):
if helpers.version_filtered(cp, ver, pv): if helpers.version_filtered(cp, ver, pv):
continue continue
url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv) url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
ret.append(( url, pv )) ret.append((url, pv))
return ret return ret
def brute_force(cpv, url): def brute_force(cpv, url):
return [] return []

45
pym/euscan/helpers.py
View File

@ -18,6 +18,7 @@ from portage import dep
from euscan import CONFIG, BLACKLIST_VERSIONS, ROBOTS_TXT_BLACKLIST_DOMAINS from euscan import CONFIG, BLACKLIST_VERSIONS, ROBOTS_TXT_BLACKLIST_DOMAINS
import euscan import euscan
def htop_vercmp(a, b): def htop_vercmp(a, b):
def fixver(v): def fixver(v):
if v in ['0.11', '0.12', '0.13']: if v in ['0.11', '0.12', '0.13']:
@ -27,12 +28,13 @@ def htop_vercmp(a, b):
return simple_vercmp(fixver(a), fixver(b)) return simple_vercmp(fixver(a), fixver(b))
VERSION_CMP_PACKAGE_QUIRKS = { VERSION_CMP_PACKAGE_QUIRKS = {
'sys-process/htop' : htop_vercmp 'sys-process/htop': htop_vercmp
} }
_v_end = '((-|_)(pre|p|beta|b|alpha|a|rc|r)\d*)' _v_end = '((-|_)(pre|p|beta|b|alpha|a|rc|r)\d*)'
_v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))' _v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))'
# Stolen from g-pypi # Stolen from g-pypi
def gentoo_mangle_version(up_pv): def gentoo_mangle_version(up_pv):
"""Convert PV to MY_PV if needed """Convert PV to MY_PV if needed
@ -146,6 +148,7 @@ def gentoo_mangle_version(up_pv):
return pv return pv
def cast_int_components(version): def cast_int_components(version):
for i, obj in enumerate(version): for i, obj in enumerate(version):
try: try:
@ -154,6 +157,7 @@ def cast_int_components(version):
pass pass
return version return version
def simple_vercmp(a, b): def simple_vercmp(a, b):
if a == b: if a == b:
return 0 return 0
@ -173,11 +177,13 @@ def simple_vercmp(a, b):
else: else:
return 1 return 1
def vercmp(package, a, b): def vercmp(package, a, b):
if package in VERSION_CMP_PACKAGE_QUIRKS: if package in VERSION_CMP_PACKAGE_QUIRKS:
return VERSION_CMP_PACKAGE_QUIRKS[package](a, b) return VERSION_CMP_PACKAGE_QUIRKS[package](a, b)
return simple_vercmp(a, b) return simple_vercmp(a, b)
def version_is_nightly(a, b): def version_is_nightly(a, b):
a = pkg_resources.parse_version(a) a = pkg_resources.parse_version(a)
b = pkg_resources.parse_version(b) b = pkg_resources.parse_version(b)
@ -188,6 +194,7 @@ def version_is_nightly(a, b):
return True return True
return False return False
def version_blacklisted(cp, version): def version_blacklisted(cp, version):
rule = None rule = None
cpv = '%s-%s' % (cp, version) cpv = '%s-%s' % (cp, version)
@ -205,6 +212,7 @@ def version_blacklisted(cp, version):
euscan.output.einfo("%s is blacklisted by rule %s" % (cpv, bv)) euscan.output.einfo("%s is blacklisted by rule %s" % (cpv, bv))
return rule is not None return rule is not None
def version_change_end_sep(version): def version_change_end_sep(version):
match = re.match('.*' + _v_end, version) match = re.match('.*' + _v_end, version)
if not match: if not match:
@ -218,6 +226,7 @@ def version_change_end_sep(version):
return None return None
return version.replace(end, newend) return version.replace(end, newend)
def version_filtered(cp, base, version, vercmp=vercmp): def version_filtered(cp, base, version, vercmp=vercmp):
if vercmp(cp, base, version) >= 0: if vercmp(cp, base, version) >= 0:
return True return True
@ -230,6 +239,7 @@ def version_filtered(cp, base, version, vercmp=vercmp):
return False return False
def generate_templates_vars(version): def generate_templates_vars(version):
ret = [] ret = []
@ -246,6 +256,7 @@ def generate_templates_vars(version):
ret.reverse() ret.reverse()
return ret return ret
def template_from_url(url, version): def template_from_url(url, version):
prefix, chunks = url.split('://') prefix, chunks = url.split('://')
chunks = chunks.split('/') chunks = chunks.split('/')
@ -261,6 +272,7 @@ def template_from_url(url, version):
return prefix + "://" + "/".join(chunks) return prefix + "://" + "/".join(chunks)
def url_from_template(url, version): def url_from_template(url, version):
components = split_version(version) components = split_version(version)
@ -270,6 +282,7 @@ def url_from_template(url, version):
return url return url
# Stolen from distutils.LooseVersion # Stolen from distutils.LooseVersion
# Used for brute force to increment the version # Used for brute force to increment the version
def split_version(version): def split_version(version):
@ -282,6 +295,7 @@ def split_version(version):
pass pass
return components return components
def join_version(components): def join_version(components):
version = "" version = ""
for i in range(len(components)): for i in range(len(components)):
@ -292,6 +306,7 @@ def join_version(components):
version += "." version += "."
return version return version
def increment_version(components, level): def increment_version(components, level):
n = len(components) n = len(components)
@ -307,6 +322,7 @@ def increment_version(components, level):
return components return components
def gen_versions(components, level): def gen_versions(components, level):
n = len(components) n = len(components)
depth = level depth = level
@ -325,6 +341,7 @@ def gen_versions(components, level):
return versions return versions
def timeout_for_url(url): def timeout_for_url(url):
if 'sourceforge' in url: if 'sourceforge' in url:
timeout = 15 timeout = 15
@ -332,13 +349,16 @@ def timeout_for_url(url):
timeout = 5 timeout = 5
return timeout return timeout
class HeadRequest(urllib2.Request): class HeadRequest(urllib2.Request):
def get_method(self): def get_method(self):
return "HEAD" return "HEAD"
""" RobotParser cache """ """ RobotParser cache """
rpcache = {} rpcache = {}
def urlallowed(url): def urlallowed(url):
if CONFIG['skip-robots-txt']: if CONFIG['skip-robots-txt']:
return True return True
@ -359,7 +379,7 @@ def urlallowed(url):
baseurl = '%s://%s' % (protocol, domain) baseurl = '%s://%s' % (protocol, domain)
robotsurl = urlparse.urljoin(baseurl, 'robots.txt') robotsurl = urlparse.urljoin(baseurl, 'robots.txt')
if rpcache.has_key(baseurl): if baseurl in rpcache:
rp = rpcache[baseurl] rp = rpcache[baseurl]
else: else:
from socket import setdefaulttimeout, getdefaulttimeout from socket import setdefaulttimeout, getdefaulttimeout
@ -379,6 +399,7 @@ def urlallowed(url):
return rp.can_fetch(CONFIG['user-agent'], url) if rp else False return rp.can_fetch(CONFIG['user-agent'], url) if rp else False
def urlopen(url, timeout=None, verb="GET"): def urlopen(url, timeout=None, verb="GET"):
if not urlallowed(url): if not urlallowed(url):
euscan.output.einfo("Url '%s' blocked by robots.txt" % url) euscan.output.einfo("Url '%s' blocked by robots.txt" % url)
@ -410,6 +431,7 @@ def urlopen(url, timeout=None, verb="GET"):
return opener.open(request, None, timeout) return opener.open(request, None, timeout)
def tryurl(fileurl, template): def tryurl(fileurl, template):
result = True result = True
@ -429,13 +451,16 @@ def tryurl(fileurl, template):
headers = fp.info() headers = fp.info()
if 'Content-disposition' in headers and basename not in headers['Content-disposition']: if 'Content-disposition' in headers and \
basename not in headers['Content-disposition']:
result = None result = None
elif 'Content-Length' in headers and headers['Content-Length'] == '0': elif 'Content-Length' in headers and headers['Content-Length'] == '0':
result = None result = None
elif 'Content-Type' in headers and 'text/html' in headers['Content-Type']: elif 'Content-Type' in headers and \
'text/html' in headers['Content-Type']:
result = None result = None
elif 'Content-Type' in headers and 'application/x-httpd-php' in headers['Content-Type']: elif 'Content-Type' in headers and \
'application/x-httpd-php' in headers['Content-Type']:
result = None result = None
elif fp.geturl() != fileurl: elif fp.geturl() != fileurl:
regex = regex_from_template(template) regex = regex_from_template(template)
@ -443,10 +468,10 @@ def tryurl(fileurl, template):
basename2 = os.path.basename(fp.geturl()) basename2 = os.path.basename(fp.geturl())
# Redirect to another (earlier?) version # Redirect to another (earlier?) version
if basename != basename2 and (re.match(regex, fp.geturl()) or re.match(baseregex, basename2)): if basename != basename2 and (re.match(regex, fp.geturl()) or \
re.match(baseregex, basename2)):
result = None result = None
if result: if result:
result = (fp.geturl(), fp.info()) result = (fp.geturl(), fp.info())
@ -459,6 +484,7 @@ def tryurl(fileurl, template):
return result return result
def regex_from_template(template): def regex_from_template(template):
# Escape # Escape
template = re.escape(template) template = re.escape(template)
@ -483,6 +509,7 @@ def regex_from_template(template):
template = template + r'/?$' template = template + r'/?$'
return template return template
def basedir_from_template(template): def basedir_from_template(template):
idx = template.find('${') idx = template.find('${')
if idx == -1: if idx == -1:
@ -494,6 +521,7 @@ def basedir_from_template(template):
return template[0:idx] return template[0:idx]
def generate_scan_paths(url): def generate_scan_paths(url):
prefix, chunks = url.split('://') prefix, chunks = url.split('://')
chunks = chunks.split('/') chunks = chunks.split('/')
@ -511,6 +539,7 @@ def generate_scan_paths(url):
return steps return steps
def parse_mirror(uri): def parse_mirror(uri):
from random import shuffle from random import shuffle
@ -526,7 +555,7 @@ def parse_mirror(uri):
return None return None
mirrorname = uri[9:eidx] mirrorname = uri[9:eidx]
path = uri[eidx+1:] path = uri[eidx + 1:]
if mirrorname in mirrors: if mirrorname in mirrors:
mirrors = mirrors[mirrorname] mirrors = mirrors[mirrorname]

26
pym/euscan/scan.py
View File

@ -4,10 +4,8 @@ import sys
import portage import portage
from portage.dbapi import porttree from portage.dbapi import porttree
from portage.output import white, yellow, turquoise, green, teal, red, EOutput
import gentoolkit.pprinter as pp import gentoolkit.pprinter as pp
from gentoolkit import errors
from gentoolkit.query import Query from gentoolkit.query import Query
from gentoolkit.eclean.search import (port_settings) from gentoolkit.eclean.search import (port_settings)
@ -17,6 +15,7 @@ from euscan import helpers
import euscan import euscan
def filter_versions(cp, versions): def filter_versions(cp, versions):
filtered = {} filtered = {}
@ -32,7 +31,8 @@ def filter_versions(cp, versions):
filtered[version] = url filtered[version] = url
return [ (cp, filtered[version], version) for version in filtered ] return [(cp, filtered[version], version) for version in filtered]
def scan_upstream_urls(cpv, urls): def scan_upstream_urls(cpv, urls):
versions = [] versions = []
@ -72,7 +72,9 @@ def scan_upstream(query):
) )
if not matches: if not matches:
sys.stderr.write(pp.warn("No package matching '%s'" % pp.pkgquery(query))) sys.stderr.write(
pp.warn("No package matching '%s'" % pp.pkgquery(query))
)
return [] return []
matches = sorted(matches) matches = sorted(matches)
@ -87,11 +89,15 @@ def scan_upstream(query):
return [] return []
if pkg.cp in BLACKLIST_PACKAGES: if pkg.cp in BLACKLIST_PACKAGES:
sys.stderr.write(pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp))) sys.stderr.write(
pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp))
)
return [] return []
if not CONFIG['quiet']: if not CONFIG['quiet']:
pp.uprint(" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name()))) pp.uprint(
" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name()))
)
pp.uprint() pp.uprint()
ebuild_path = pkg.ebuild_path() ebuild_path = pkg.ebuild_path()
@ -104,8 +110,8 @@ def scan_upstream(query):
cpv = pkg.cpv cpv = pkg.cpv
metadata = { metadata = {
"EAPI" : port_settings["EAPI"], "EAPI": port_settings["EAPI"],
"SRC_URI" : pkg.environment("SRC_URI", False), "SRC_URI": pkg.environment("SRC_URI", False),
} }
use = frozenset(port_settings["PORTAGE_USE"].split()) use = frozenset(port_settings["PORTAGE_USE"].split())
try: try:
@ -113,7 +119,9 @@ def scan_upstream(query):
aalist = porttree._parse_uri_map(cpv, metadata) aalist = porttree._parse_uri_map(cpv, metadata)
except Exception as e: except Exception as e:
sys.stderr.write(pp.warn("%s\n" % str(e))) sys.stderr.write(pp.warn("%s\n" % str(e)))
sys.stderr.write(pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv))) sys.stderr.write(
pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv))
)
return [] return []
if "mirror" in portage.settings.features: if "mirror" in portage.settings.features:

4
setup.py
View File

@ -81,8 +81,8 @@ setup(
'https://github.com/iksaif/euscan/tarball/' + 'https://github.com/iksaif/euscan/tarball/' +
('master' if __version__ == '9999' else ('euscan-%s' % __version__)) ('master' if __version__ == '9999' else ('euscan-%s' % __version__))
), ),
install_requires=['Django==1.3.1', 'django-annoying', 'South', install_requires=['Django==1.4', 'django-annoying', 'South',
'django-piston', 'matplotlib'], 'django-piston', 'matplotlib', 'BeautifulSoup'],
package_dir={'': 'pym'}, package_dir={'': 'pym'},
packages=packages, packages=packages,
package_data={}, package_data={},