src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #2 from volpino/master

Browse Source 
Cleaning up and updating requirements
This commit is contained in:
Corentin Chary 2012-04-29 12:42:14 -07:00
commit 1a40cf7bcc
46 changed files with 796 additions and 451 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README
View File

@ -88,6 +88,15 @@ tree to avoid messing with your system.
### Installation
Install requirements from PyPI using
$ python setup.py develop
Extra dependencies:
* portage python api
* rrdtool[python]
Like any django web app, just start by editing settings.py and then run
these two commands.

96
bin/euscan
View File

@ -34,11 +34,12 @@ from euscan.scan import scan_upstream
""" Globals """
def setupSignals():
""" This block ensures that ^C interrupts are handled quietly. """
import signal
def exithandler(signum,frame):
def exithandler(signum, frame):
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
print ()
@ -51,10 +52,10 @@ def setupSignals():
def printVersion():
"""Output the version info."""
print( "%s (%s) - %s" \
print("%s (%s) - %s" \
% (__productname__, __version__, __description__))
print()
print("Author: %s <%s>" % (__author__,__email__))
print("Author: %s <%s>" % (__author__, __email__))
print("Copyright 2011 Gentoo Foundation")
print("Distributed under the terms of the GNU General Public License v2")
@ -68,45 +69,49 @@ def printUsage(_error=None, help=None):
out = sys.stderr
if not _error in ('global-options', 'packages',):
_error = None
if not _error and not help: help = 'all'
if not _error and not help:
help = 'all'
if _error in ('global-options',):
print( pp.error("Wrong option on command line."), file=out)
print( file=out)
print(pp.error("Wrong option on command line."), file=out)
print(file=out)
if _error in ('packages',):
print( pp.error("You need to specify exactly one package."), file=out)
print( file=out)
print( white("Usage:"), file=out)
print(pp.error("You need to specify exactly one package."), file=out)
print(file=out)
print(white("Usage:"), file=out)
if _error in ('global-options', 'packages',) or help == 'all':
print( " "+turquoise(__productname__),
print(" " + turquoise(__productname__),
yellow("[options]"),
green("<package> [<package> [...]]"), file=out)
if _error in ('global-options',) or help == 'all':
print( " "+turquoise(__productname__),
print(" " + turquoise(__productname__),
yellow("[--help, --version]"), file=out)
print(file=out)
if _error in ('global-options',) or help:
print( "Available ", yellow("options")+":", file=out)
print( yellow(" -C, --nocolor")+
" - turn off colors on output", file=out)
print( yellow(" -q, --quiet")+
" - be as quiet as possible", file=out)
print( yellow(" -h, --help")+ \
" - display the help screen", file=out)
print( yellow(" -V, --version")+
" - display version info", file=out)
print( file=out)
print( yellow(" -1, --oneshot")+
" - stop as soon as a new version is found", file=out)
print( yellow(" -b, --brute-force=<level>")+
" - define the brute force "+yellow("<level>")+" (default: 2)\n" +
" " * 29 + "bigger levels will generate more versions numbers\n" +
" " * 29 + "0 means disabled", file=out)
print( file=out)
print("Available ", yellow("options") + ":", file=out)
print(yellow(" -C, --nocolor") +
" - turn off colors on output", file=out)
print(yellow(" -q, --quiet") +
" - be as quiet as possible", file=out)
print(yellow(" -h, --help") +
" - display the help screen", file=out)
print(yellow(" -V, --version") +
" - display version info", file=out)
print(file=out)
print(yellow(" -1, --oneshot") +
" - stop as soon as a new version is found",
file=out)
print(yellow(" -b, --brute-force=<level>") +
" - define the brute force " + yellow("<level>") +
" (default: 2)\n" +
" " * 29 + "bigger levels will generate more versions numbers\n" +
" " * 29 + "0 means disabled", file=out)
print(file=out)
if _error in ('packages',) or help:
print( green(" package")+
" - the packages (or ebuilds) you want to scan", file=out)
print( file=out)
print(green(" package") +
" - the packages (or ebuilds) you want to scan",
file=out)
print(file=out)
'''print( "More detailed instruction can be found in",
turquoise("`man %s`" % __productname__), file=out)'''
@ -115,6 +120,7 @@ class ParseArgsException(Exception):
"""For parseArgs() -> main() communications."""
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
@ -130,7 +136,7 @@ def parseArgs():
return_code = True
for o, a in opts:
if o in ("-h", "--help"):
raise ParseArgsException('help')
raise ParseArgsException('help')
elif o in ("-V", "--version"):
raise ParseArgsException('version')
elif o in ("-C", "--nocolor"):
@ -151,7 +157,7 @@ def parseArgs():
return return_code
' here are the different allowed command line options (getopt args) '
getopt_options = {'short':{}, 'long':{}}
getopt_options = {'short': {}, 'long': {}}
getopt_options['short']['global'] = "hVCqv1b:"
getopt_options['long']['global'] = ["help", "version", "nocolor", "quiet",
"verbose", "oneshot", "brute-force="]
@ -164,19 +170,20 @@ def parseArgs():
try:
opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts)
except:
raise ParseArgsException(opts_mode+'-options')
raise ParseArgsException(opts_mode + '-options')
' set options accordingly '
optionSwitch(opts)
if len(args) < 1:
raise ParseArgsException('packages')
raise ParseArgsException('packages')
return args
def main():
"""Parse command line and execute all actions."""
CONFIG['nocolor'] = (port_settings["NOCOLOR"] in ('yes','true')
CONFIG['nocolor'] = (port_settings["NOCOLOR"] in ('yes', 'true')
or not sys.stdout.isatty())
if CONFIG['nocolor']:
pp.output.nocolor()
@ -212,11 +219,16 @@ def main():
for candidate in pkgs:
print(candidate)
from os.path import basename # To get the short name
from os.path import basename # To get the short name
print(file=sys.stderr)
print(pp.error("The short ebuild name '%s' is ambiguous. Please specify" % basename(pkgs[0])),
file=sys.stderr, end="")
print(
pp.error(
"The short ebuild name '%s' is ambiguous. Please specify" \
% basename(pkgs[0])
),
file=sys.stderr, end=""
)
pp.die(1, "one of the above fully-qualified ebuild names instead.")
except GentoolkitException as err:
pp.die(1, '%s: %s' % (package, str(err)))
@ -233,12 +245,12 @@ def main():
+ pp.path(" %s" % url))
else:
print (pp.cpv("%s-%s" % (cp, version))
+ ": " + pp.path(url))
+ ": " + pp.path(url))
if not len(ret) and not CONFIG['quiet']:
print (pp.warn("Didn't find any new version, "
+ "check package's homepage for "
+ "more informations"));
+ "more informations"))
if __name__ == "__main__":
@ -246,6 +258,6 @@ if __name__ == "__main__":
setupSignals()
main()
except KeyboardInterrupt:
print( "Aborted.")
print("Aborted.")
sys.exit(errno.EINTR)
sys.exit(0)

5
euscanwww/djeuscan/admin.py
View File

@ -1,11 +1,14 @@
from models import *
from models import Package, Version, VersionLog, EuscanResult, Log, WorldLog, \
CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer
from django.contrib import admin
admin.site.register(Herd)
admin.site.register(Maintainer)
class PackageAdmin(admin.ModelAdmin):
search_fields = ('category', 'name')
admin.site.register(Package, PackageAdmin)
admin.site.register(Version)

17
euscanwww/djeuscan/api/emitters.py
View File

@ -1,15 +1,16 @@
from piston.emitters import Emitter, XMLEmitter
from piston.emitters import XMLEmitter
class EuscanXMLEmitter(XMLEmitter):
_parent = []
_known_parents = {
'vlog' : 'version',
'herds' : 'herd',
'maintainers' : 'maintainer',
'packaged' : 'version',
'upstream' : 'version',
'packages' : 'package',
'categories' : 'category'
'vlog': 'version',
'herds': 'herd',
'maintainers': 'maintainer',
'packaged': 'version',
'upstream': 'version',
'packages': 'package',
'categories': 'category'
}
def _push_parent(self, parent):

87
euscanwww/djeuscan/api/handlers.py
View File

@ -1,24 +1,26 @@
from piston.handler import AnonymousBaseHandler, BaseHandler
from piston.utils import rc, HttpStatusCode
from piston.handler import AnonymousBaseHandler
from piston.utils import rc
from django.db.models import Sum, Max
from django.core.exceptions import ObjectDoesNotExist
from django.forms.models import model_to_dict
from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, VersionLog
from djeuscan.forms import WorldForm, PackagesForm
from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
VersionLog
# replace default XMLEmitter with ours
from piston.emitters import Emitter
from emitters import EuscanXMLEmitter
Emitter.register('xml', EuscanXMLEmitter, 'text/xml; charset=utf-8')
def xint(i):
try:
return int(i)
except:
return 0
def renameFields(vqs, fields):
ret = []
for n in vqs:
@ -29,6 +31,7 @@ def renameFields(vqs, fields):
ret.append(n)
return ret
class catch_and_return(object):
def __init__(self, err, response):
self.err = err
@ -42,6 +45,7 @@ class catch_and_return(object):
return self.response
return wrapper
# /api/1.0/
class RootHandler(AnonymousBaseHandler):
allowed_methods = ('GET',)
@ -49,52 +53,70 @@ class RootHandler(AnonymousBaseHandler):
def read(self, request):
return {'api-version': '1.0'}
# /api/1.0/statistics
class StatisticsHandler(AnonymousBaseHandler):
allowed_methods = ('GET',)
def read(self, request):
_aggr = Package.objects.aggregate
data = {}
data['n_packaged'] = xint(Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum'])
data['n_overlay'] = xint(Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum'])
data['n_versions'] = xint(Package.objects.aggregate(Sum('n_versions'))['n_versions__sum'])
data['n_upstream'] = data['n_versions'] - data['n_packaged'] - data['n_overlay']
data['n_packaged'] = xint(_aggr(Sum('n_packaged'))['n_packaged__sum'])
data['n_overlay'] = xint(_aggr(Sum('n_overlay'))['n_overlay__sum'])
data['n_versions'] = xint(_aggr(Sum('n_versions'))['n_versions__sum'])
data['n_upstream'] = data['n_versions'] - data['n_packaged'] - \
data['n_overlay']
data['n_packages'] = Package.objects.count()
data['n_herds'] = Herd.objects.count()
data['n_maintainers'] = Maintainer.objects.count()
data['last_scan'] = EuscanResult.objects.get(id=EuscanResult.objects.aggregate(Max('id'))['id__max']).datetime
data['last_scan'] = EuscanResult.objects.get(
id=EuscanResult.objects.aggregate(Max('id'))['id__max']
).datetime
return data
# /api/1.0/maintainers
class MaintainersHandler(AnonymousBaseHandler):
allowed_methods = ('GET',)
def read(self, request):
maintainers = Package.objects.filter(maintainers__isnull=False)
maintainers = maintainers.values('maintainers__id', 'maintainers__name', 'maintainers__email')
maintainers = maintainers.values(
'maintainers__id',
'maintainers__name',
'maintainers__email'
)
maintainers = maintainers.annotate(n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
maintainers = renameFields(maintainers, [('maintainers__id', 'id'),
('maintainers__name', 'name'),
('maintainers__email', 'email')])
return { 'maintainers' : maintainers }
maintainers = renameFields(
maintainers,
[('maintainers__id', 'id'),
('maintainers__name', 'name'),
('maintainers__email', 'email')]
)
return {'maintainers': maintainers}
# /api/1.0/herds
class HerdsHandler(AnonymousBaseHandler):
allowed_methods = ('GET',)
def read(self, request):
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of 'INNER JOIN'
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN'
# instead of 'INNER JOIN'
herds = Package.objects.filter(herds__isnull=False)
herds = herds.values('herds__herd').annotate(n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
herds = herds.values('herds__herd').annotate(
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')
)
herds = renameFields(herds, [('herds__herd', 'herd')])
return { 'herds' : herds }
return {'herds': herds}
# /api/1.0/categories
class CategoriesHandler(AnonymousBaseHandler):
@ -106,7 +128,8 @@ class CategoriesHandler(AnonymousBaseHandler):
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
return { 'categories' : categories }
return {'categories': categories}
# /api/1.0/packages/by-maintainer/
# /api/1.0/packages/by-category/
@ -125,17 +148,21 @@ class PackagesHandler(AnonymousBaseHandler):
if 'category' in kwargs:
packages = Package.objects.filter(category=kwargs['category'])
data = { 'category' : kwargs['category'] }
data = {'category': kwargs['category']}
elif 'herd' in kwargs:
herd = Herd.objects.get(herd=kwargs['herd'])
packages = Package.objects.filter(herds__id=herd.id)
data = { 'herd' : herd }
data = {'herd': herd}
elif 'maintainer_id' in kwargs:
maintainer = Maintainer.objects.get(id=kwargs['maintainer_id'])
packages = Package.objects.filter(maintainers__id=maintainer.id)
data = { 'maintainer' : maintainer }
data = {'maintainer': maintainer}
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
packages = packages.select_related(
'last_version_gentoo',
'last_version_overlay',
'last_version_upstream'
)
data['packages'] = packages
if not data:
@ -143,6 +170,7 @@ class PackagesHandler(AnonymousBaseHandler):
return data
# /api/1.0/package/
class PackageHandler(AnonymousBaseHandler):
allowed_methods = ('GET',)
@ -152,7 +180,8 @@ class PackageHandler(AnonymousBaseHandler):
package = Package.objects.get(category=category, name=package)
package.homepages = package.homepage.split(' ')
versions = Version.objects.filter(package=package)
log = EuscanResult.objects.filter(package=package).order_by('-datetime')[:1]
log = EuscanResult.objects.filter(package=package).\
order_by('-datetime')[:1]
log = log[0] if log else None
vlog = VersionLog.objects.filter(package=package).order_by('-id')
@ -166,7 +195,8 @@ class PackageHandler(AnonymousBaseHandler):
version_log = []
for v in vlog:
v = model_to_dict(v, ['version', 'revision', 'slot', 'overlay', 'datetime', 'action'])
v = model_to_dict(v, ['version', 'revision', 'slot', 'overlay',
'datetime', 'action'])
if v['action'] == VersionLog.VERSION_ADDED:
v['action'] = 'added'
if v['action'] == VersionLog.VERSION_REMOVED:
@ -177,7 +207,10 @@ class PackageHandler(AnonymousBaseHandler):
packaged = []
for version in versions:
unpackaged = not version.packaged
version = model_to_dict(version, ['version', 'revision', 'slot', 'overlay', 'urls'])
version = model_to_dict(
version,
['version', 'revision', 'slot', 'overlay', 'urls']
)
if unpackaged:
upstream.append(version)
else:

32
euscanwww/djeuscan/api/urls.py
View File

@ -1,7 +1,8 @@
from django.conf.urls.defaults import *
from django.conf.urls.defaults import url, patterns
from piston.resource import Resource
from handlers import *
from handlers import RootHandler, StatisticsHandler, HerdsHandler, \
CategoriesHandler, MaintainersHandler, PackagesHandler, PackageHandler
root_handler = Resource(handler=RootHandler)
statistics_handler = Resource(handler=StatisticsHandler)
@ -12,16 +13,25 @@ packages_handler = Resource(handler=PackagesHandler)
package_handler = Resource(handler=PackageHandler)
urlpatterns = patterns('djeuscan.api.views',
url(r'^1.0/statistics\.(?P<emitter_format>.+)$', statistics_handler, name='api.views.statistics'),
url(r'^1.0/herds\.(?P<emitter_format>.+)$', herds_handler, name='api.views.herds'),
url(r'^1.0/categories\.(?P<emitter_format>.+)$', categories_handler, name='api.views.categories'),
url(r'^1.0/maintainers\.(?P<emitter_format>.+)$', maintainers_handler, name='api.views.maintainers'),
url(r'^1.0/statistics\.(?P<emitter_format>.+)$', statistics_handler,
name='api.views.statistics'),
url(r'^1.0/herds\.(?P<emitter_format>.+)$', herds_handler,
name='api.views.herds'),
url(r'^1.0/categories\.(?P<emitter_format>.+)$', categories_handler,
name='api.views.categories'),
url(r'^1.0/maintainers\.(?P<emitter_format>.+)$', maintainers_handler,
name='api.views.maintainers'),
url(r'^1.0/packages/by-maintainer/(?P<maintainer_id>\d+)\.(?P<emitter_format>.+)$', packages_handler, name='api.views.packages'),
url(r'^1.0/packages/by-herd/(?P<herd>[\@\{\}\w+.-]*)\.(?P<emitter_format>.+)?$', packages_handler, name='api.views.packages'),
url(r'^1.0/packages/by-category/(?P<category>[\w+][\w+.-]*)\.(?P<emitter_format>.+)?$', packages_handler, name='api.views.packages'),
url(r'^1.0/packages/by-maintainer/(?P<maintainer_id>\d+)\.(?P<emitter_format>.+)$',
packages_handler, name='api.views.packages'),
url(r'^1.0/packages/by-herd/(?P<herd>[\@\{\}\w+.-]*)\.(?P<emitter_format>.+)?$',
packages_handler, name='api.views.packages'),
url(r'^1.0/packages/by-category/(?P<category>[\w+][\w+.-]*)\.(?P<emitter_format>.+)?$',
packages_handler, name='api.views.packages'),
url(r'^1.0/package/(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)\.(?P<emitter_format>.+)$', package_handler, name='api.views.package'),
url(r'^1.0/package/(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)\.(?P<emitter_format>.+)$',
package_handler, name='api.views.package'),
url(r'^1.0/api\.(?P<emitter_format>.+)$', root_handler, name='api.views.root'),
url(r'^1.0/api\.(?P<emitter_format>.+)$',
root_handler, name='api.views.root'),
)

147
euscanwww/djeuscan/charts.py
View File

@ -3,14 +3,12 @@ import time
from euscanwww import settings
from django.db.models import F, Sum, Max
from djeuscan.models import Version, Package, Herd, Maintainer
from djeuscan.models import CategoryLog
from django.db.models import F, Sum
from djeuscan.models import Package
import rrdtool
import pylab
import matplotlib
CHARTS_ROOT = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts')
CHARTS_URL = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts')
@ -21,12 +19,14 @@ pylab.rcParams['axes.titlesize'] = 10.0
pylab.rcParams['xtick.labelsize'] = 8.0
pylab.rcParams['legend.fontsize'] = 8.0
def xint(i):
try:
return int(i)
except:
return 0
def chart_alive(name):
path = os.path.join(CHARTS_ROOT, name)
if not os.path.exists(path):
@ -37,6 +37,7 @@ def chart_alive(name):
return False
return True
def rrd_name(**kwargs):
name = ""
@ -51,6 +52,7 @@ def rrd_name(**kwargs):
return name
def chart_name(name, **kwargs):
name = name.replace('_', '-')
@ -67,6 +69,7 @@ def chart_name(name, **kwargs):
return name + ".png"
def getpackages(**kwargs):
packages = Package.objects
@ -79,6 +82,7 @@ def getpackages(**kwargs):
return packages
def cached_pylab_chart(f):
def new_f(*args, **kwds):
name = chart_name(f.func_name, **kwds)
@ -93,14 +97,16 @@ def cached_pylab_chart(f):
new_f.func_name = f.func_name
return new_f
@cached_pylab_chart
def pie_versions(**kwargs):
n_packaged = xint(getpackages(**kwargs).aggregate(Sum('n_packaged'))['n_packaged__sum'])
n_overlay = xint(getpackages(**kwargs).aggregate(Sum('n_overlay'))['n_overlay__sum'])
n_versions = xint(getpackages(**kwargs).aggregate(Sum('n_versions'))['n_versions__sum'])
gpk = getpackages(**kwargs)
n_packaged = xint(gpk.aggregate(Sum('n_packaged'))['n_packaged__sum'])
n_overlay = xint(gpk.aggregate(Sum('n_overlay'))['n_overlay__sum'])
n_versions = xint(gpk.aggregate(Sum('n_versions'))['n_versions__sum'])
n_upstream = n_versions - n_packaged - n_overlay
pylab.figure(1, figsize=(3.5,3.5))
pylab.figure(1, figsize=(3.5, 3.5))
if n_overlay:
labels = 'Gentoo', 'Overlays', 'Upstream'
@ -111,35 +117,43 @@ def pie_versions(**kwargs):
fracs = [n_packaged, n_upstream]
colors = '#008000', '#FF0000'
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', shadow=True)
pylab.title('Versions', bbox={'facecolor':'0.8', 'pad':5})
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%',
shadow=True)
pylab.title('Versions', bbox={'facecolor': '0.8', 'pad': 5})
@cached_pylab_chart
def pie_packages(**kwargs):
n_packages = getpackages(**kwargs).count()
n_packages_uptodate_main = getpackages(**kwargs).filter(n_versions=F('n_packaged')).count()
n_packages_uptodate_all = getpackages(**kwargs).filter(n_versions=F('n_packaged') + F('n_overlay')).count()
gpk = getpackages(**kwargs)
n_packages = gpk.count()
n_packages_uptodate_main = gpk.filter(n_versions=F('n_packaged')).count()
n_packages_uptodate_all = gpk.filter(n_versions=F('n_packaged') + \
F('n_overlay')).count()
n_packages_outdated = n_packages - n_packages_uptodate_all
n_packages_uptodate_ovl = n_packages_uptodate_all - n_packages_uptodate_main
n_packages_uptodate_ovl = n_packages_uptodate_all - \
n_packages_uptodate_main
pylab.figure(1, figsize=(3.5,3.5))
pylab.figure(1, figsize=(3.5, 3.5))
if n_packages_uptodate_ovl:
labels = 'Ok (gentoo)', 'Ok (overlays)', 'Outdated'
fracs = [n_packages_uptodate_main, n_packages_uptodate_ovl, n_packages_outdated]
fracs = [n_packages_uptodate_main, n_packages_uptodate_ovl,
n_packages_outdated]
colors = '#008000', '#0B17FD', '#FF0000'
else:
labels = 'Ok (gentoo)', 'Outdated'
fracs = [n_packages_uptodate_main, n_packages_outdated]
colors = '#008000', '#FF0000'
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', shadow=True)
pylab.title('Packages', bbox={'facecolor':'0.8', 'pad':5})
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%',
shadow=True)
pylab.title('Packages', bbox={'facecolor': '0.8', 'pad': 5})
def rrd_path(name):
return str(os.path.join(settings.RRD_ROOT, name + '.rrd'))
def rrd_create(name, start):
path = rrd_path(name)
if os.path.exists(path):
@ -156,13 +170,17 @@ def rrd_create(name, start):
'RRA:AVERAGE:0.5:5:200',
'RRA:AVERAGE:0.5:10:200')
def rrd_update(name, datetime, values):
now = time.mktime(datetime.timetuple())
rrd_create(name, now)
rrdtool.update(rrd_path(name),
'%d:%d:%d:%d:%d:%d:%d' % \
(now, values.n_packages_gentoo, values.n_packages_overlay, values.n_packages_outdated, \
values.n_versions_gentoo, values.n_versions_overlay, values.n_versions_upstream))
rrdtool.update(
rrd_path(name),
'%d:%d:%d:%d:%d:%d:%d' % \
(now, values.n_packages_gentoo, values.n_packages_overlay,
values.n_packages_outdated, values.n_versions_gentoo,
values.n_versions_overlay, values.n_versions_upstream)
)
"""
@ -175,6 +193,7 @@ def rrd_update(name, datetime, values):
[-J|--alt-autoscale-min]
"""
def cached_rrd_chart(f):
def new_f(*args, **kwds):
if 'period' not in kwds:
@ -215,47 +234,55 @@ def cached_rrd_chart(f):
new_f.func_name = f.func_name
return new_f
@cached_rrd_chart
def packages(**kwargs):
rrdtool.graph(str(kwargs['path']),
'--imgformat', 'PNG',
'--width', kwargs['width'],
'--height', kwargs['height'],
kwargs['graph-mode'],
'--color', 'CANVAS#FFFFFF00',
'--color', 'BACK#FFFFFF00',
rrdtool.graph(
str(kwargs['path']),
'--imgformat', 'PNG',
'--width', kwargs['width'],
'--height', kwargs['height'],
kwargs['graph-mode'],
'--color', 'CANVAS#FFFFFF00',
'--color', 'BACK#FFFFFF00',
'--start', kwargs['start'],
'--end', kwargs['end'],
'--vertical-label', kwargs['vertical-label'],
'--title', kwargs['title'],
'--lower-limit', '0',
'DEF:n_packages_gentoo=%s:n_packages_gentoo:AVERAGE' % (kwargs['rrd']),
'DEF:n_packages_overlay=%s:n_packages_overlay:AVERAGE' % \
(kwargs['rrd']),
'DEF:n_packages_outdated=%s:n_packages_outdated:AVERAGE' % \
(kwargs['rrd']),
'LINE1.25:n_packages_gentoo#008000:Gentoo',
'LINE1.25:n_packages_overlay#0B17FD:Overlay',
'LINE1.25:n_packages_outdated#FF0000:Outdated'
)
'--start', kwargs['start'],
'--end', kwargs['end'],
'--vertical-label', kwargs['vertical-label'],
'--title', kwargs['title'],
'--lower-limit', '0',
'DEF:n_packages_gentoo=%s:n_packages_gentoo:AVERAGE' % (kwargs['rrd']),
'DEF:n_packages_overlay=%s:n_packages_overlay:AVERAGE' % (kwargs['rrd']),
'DEF:n_packages_outdated=%s:n_packages_outdated:AVERAGE' % (kwargs['rrd']),
'LINE1.25:n_packages_gentoo#008000:Gentoo',
'LINE1.25:n_packages_overlay#0B17FD:Overlay',
'LINE1.25:n_packages_outdated#FF0000:Outdated')
@cached_rrd_chart
def versions(**kwargs):
rrdtool.graph(str(kwargs['path']),
'--imgformat', 'PNG',
'--width', kwargs['width'],
'--height', kwargs['height'],
kwargs['graph-mode'],
'--color', 'CANVAS#FFFFFF00',
'--color', 'BACK#FFFFFF00',
'--start', kwargs['start'],
'--end', kwargs['end'],
'--vertical-label', kwargs['vertical-label'],
'--title', kwargs['title'],
'--lower-limit', '0',
'DEF:n_versions_gentoo=%s:n_versions_gentoo:AVERAGE' % (kwargs['rrd']),
'DEF:n_versions_overlay=%s:n_versions_overlay:AVERAGE' % (kwargs['rrd']),
'DEF:n_versions_outdated=%s:n_versions_upstream:AVERAGE' % (kwargs['rrd']),
'LINE1.25:n_versions_gentoo#008000:Gentoo',
'LINE1.25:n_versions_overlay#0B17FD:Overlay',
'LINE1.25:n_versions_outdated#FF0000:Outdated')
rrdtool.graph(
str(kwargs['path']),
'--imgformat', 'PNG',
'--width', kwargs['width'],
'--height', kwargs['height'],
kwargs['graph-mode'],
'--color', 'CANVAS#FFFFFF00',
'--color', 'BACK#FFFFFF00',
'--start', kwargs['start'],
'--end', kwargs['end'],
'--vertical-label', kwargs['vertical-label'],
'--title', kwargs['title'],
'--lower-limit', '0',
'DEF:n_versions_gentoo=%s:n_versions_gentoo:AVERAGE' % (kwargs['rrd']),
'DEF:n_versions_overlay=%s:n_versions_overlay:AVERAGE' % \
(kwargs['rrd']),
'DEF:n_versions_outdated=%s:n_versions_upstream:AVERAGE' % \
(kwargs['rrd']),
'LINE1.25:n_versions_gentoo#008000:Gentoo',
'LINE1.25:n_versions_overlay#0B17FD:Overlay',
'LINE1.25:n_versions_outdated#FF0000:Outdated'
)

32
euscanwww/djeuscan/feeds.py
View File

@ -1,11 +1,10 @@
from django.contrib.syndication.views import Feed, FeedDoesNotExist
from django.shortcuts import get_object_or_404
from django.http import Http404
from django.utils.feedgenerator import Atom1Feed
from django.core.urlresolvers import reverse
from djeuscan.models import Version, Package, Herd, Maintainer, VersionLog
from djeuscan.views import *
from djeuscan.models import Package, Herd, Maintainer, VersionLog
class BaseFeed(Feed):
feed_type = Atom1Feed
@ -18,8 +17,8 @@ class BaseFeed(Feed):
def item_description(self, vlog):
if vlog.overlay:
txt = 'Version %s-%s [%s] of package %s ' % (vlog.version, vlog.revision,
vlog.slot, vlog.package)
txt = 'Version %s-%s [%s] of package %s ' % \
(vlog.version, vlog.revision, vlog.slot, vlog.package)
else:
txt = 'Version %s of package %s ' % (vlog.version, vlog.package)
if vlog.action == vlog.VERSION_REMOVED:
@ -36,8 +35,10 @@ class BaseFeed(Feed):
return txt
def item_link(self, vlog):
kwargs = {'category' : vlog.package.category, 'package' : vlog.package.name }
return reverse('djeuscan.views.package', kwargs=kwargs) + '#' + vlog.tag()
kwargs = {'category': vlog.package.category,
'package': vlog.package.name}
return "%s#%s" % (reverse('djeuscan.views.package', kwargs=kwargs),
vlog.tag())
def item_pubdate(self, vlog):
return vlog.datetime
@ -45,18 +46,20 @@ class BaseFeed(Feed):
def item_categories(self, vlog):
return [vlog.package.category]
class GlobalFeed(BaseFeed):
title = "euscan"
link = "/"
description = "Last euscan changes"
def categories(self):
categories = Package.objects.values('category').distinct();
return [ category['category'] for category in categories ]
categories = Package.objects.values('category').distinct()
return [category['category'] for category in categories]
def items(self):
return VersionLog.objects.order_by('-id')[:250]
class PackageFeed(BaseFeed):
feed_type = Atom1Feed
@ -67,7 +70,8 @@ class PackageFeed(BaseFeed):
return "%s" % package
def link(self, package):
return reverse('djeuscan.views.package', args=[package.category, package.name])
return reverse('djeuscan.views.package', args=[package.category,
package.name])
def description(self, package):
return package.description
@ -78,6 +82,7 @@ class PackageFeed(BaseFeed):
def item_description(self, vlog):
return ''
class MaintainerFeed(BaseFeed):
feed_type = Atom1Feed
@ -91,12 +96,14 @@ class MaintainerFeed(BaseFeed):
return "Last changes for %s" % maintainer
def link(self, maintainer):
return reverse('djeuscan.views.maintainer', kwargs={'maintainer_id' : maintainer.id})
return reverse('djeuscan.views.maintainer',
kwargs={'maintainer_id': maintainer.id})
def items(self, maintainer):
q = VersionLog.objects.filter(package__maintainers__id=maintainer.id)
return q.order_by('-id')[:50]
class HerdFeed(BaseFeed):
feed_type = Atom1Feed
@ -110,12 +117,13 @@ class HerdFeed(BaseFeed):
return "Last changes for %s" % herd
def link(self, herd):
return reverse('djeuscan.views.herd', kwargs={'herd' : herd.herd})
return reverse('djeuscan.views.herd', kwargs={'herd': herd.herd})
def items(self, herd):
q = VersionLog.objects.filter(package__herds__id=herd.id)
return q.order_by('-id')[:100]
class CategoryFeed(BaseFeed):
feed_type = Atom1Feed

2
euscanwww/djeuscan/forms.py
View File

@ -1,7 +1,9 @@
from django import forms
class WorldForm(forms.Form):
world = forms.FileField()
class PackagesForm(forms.Form):
packages = forms.CharField(widget=forms.Textarea)

3
euscanwww/djeuscan/management/commands/list-packages.py
View File

@ -1,8 +1,9 @@
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import Package
class Command(BaseCommand):
_overlays = {}
help = 'List packages'

13
euscanwww/djeuscan/management/commands/regen-rrds.py
View File

@ -1,11 +1,8 @@
import datetime
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from djeuscan import charts
class Command(BaseCommand):
_overlays = {}
help = 'Regenerate rrd database'
@ -15,10 +12,12 @@ class Command(BaseCommand):
charts.rrd_update('world', wlog.datetime, wlog)
for clog in CategoryLog.objects.all():
charts.rrd_update('category-%s' % clog.category, clog.datetime, clog)
charts.rrd_update('category-%s' % clog.category,
clog.datetime, clog)
for hlog in HerdLog.objects.all():
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
for mlog in MaintainerLog.objects.all():
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, mlog.datetime, mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
mlog.datetime, mlog)

53
euscanwww/djeuscan/management/commands/scan-metadata.py
View File

@ -1,19 +1,15 @@
import subprocess
import portage
import sys
import os
import re
from portage import versions
from optparse import make_option
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import Package, Herd, Maintainer
from gentoolkit.query import Query
from gentoolkit.errors import GentoolkitFatalError
class Command(BaseCommand):
_overlays = {}
@ -53,16 +49,20 @@ class Command(BaseCommand):
)
if not matches:
sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % query))
sys.stderr.write(
self.style.ERROR("Unknown package '%s'\n" % query)
)
return
matches = sorted(matches)
matches = sorted(matches)
pkg = matches.pop()
if '9999' in pkg.version and len(matches):
if '9999' in pkg.version and len(matches):
pkg = matches.pop()
if not obj:
obj, created = Package.objects.get_or_create(category=pkg.category, name=pkg.name)
obj, created = Package.objects.get_or_create(
category=pkg.category, name=pkg.name
)
else:
created = False
@ -70,22 +70,34 @@ class Command(BaseCommand):
obj.homepage = pkg.environment("HOMEPAGE")
obj.description = pkg.environment("DESCRIPTION")
except GentoolkitFatalError, err:
sys.stderr.write(self.style.ERROR("Gentoolkit fatal error: '%s'\n" % str(err)))
sys.stderr.write(
self.style.ERROR(
"Gentoolkit fatal error: '%s'\n" % str(err)
)
)
if created and not self.options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
if pkg.metadata:
herds = dict([(herd[0], herd) for herd in pkg.metadata.herds(True)])
maintainers = dict([(m.email, m) for m in pkg.metadata.maintainers()])
herds = dict(
[(herd[0], herd) for herd in pkg.metadata.herds(True)]
)
maintainers = dict(
[(m.email, m) for m in pkg.metadata.maintainers()]
)
existing_herds = [h.herd for h in obj.herds.all()]
new_herds = set(herds.keys()).difference(existing_herds)
old_herds = set(existing_herds).difference(herds.keys())
existing_maintainers = [m.email for m in obj.maintainers.all()]
new_maintainers = set(maintainers.keys()).difference(existing_maintainers)
old_maintainers = set(existing_maintainers).difference(maintainers.keys())
new_maintainers = set(
maintainers.keys()).difference(existing_maintainers
)
old_maintainers = set(
existing_maintainers).difference(maintainers.keys()
)
for herd in obj.herds.all():
if herd.herd in old_herds:
@ -101,7 +113,9 @@ class Command(BaseCommand):
for maintainer in new_maintainers:
maintainer = maintainers[maintainer]
maintainer = self.store_maintainer(maintainer.name, maintainer.email)
maintainer = self.store_maintainer(
maintainer.name, maintainer.email
)
obj.maintainers.add(maintainer)
obj.save()
@ -131,9 +145,12 @@ class Command(BaseCommand):
if created:
if not self.options['quiet']:
sys.stdout.write('+ [m] %s <%s>\n' % (name.encode('utf-8'), email))
sys.stdout.write(
'+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
)
if not maintainer.name or name not in [maintainer.name, email, '{nil}']:
if not maintainer.name or \
name not in [maintainer.name, email, '{nil}']:
maintainer.name = name
maintainer.save()

94
euscanwww/djeuscan/management/commands/scan-portage.py
View File

@ -4,13 +4,13 @@ import sys
import os
import re
from portage import versions
from optparse import make_option
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import Package, Version, VersionLog
class Command(BaseCommand):
_overlays = {}
@ -39,7 +39,8 @@ class Command(BaseCommand):
action='store_true',
dest='prefetch',
default=False,
help='Prefetch all versions and packages from DB to speedup full scan process.'),
help=('Prefetch all versions and packages from DB to '
'speedup full scan process.')),
make_option('--quiet',
action='store_true',
dest='quiet',
@ -49,7 +50,7 @@ class Command(BaseCommand):
args = '[package package ...]'
help = 'Scans portage tree and fills database'
_cache = {'packages' : {}, 'versions' : {}}
_cache = {'packages': {}, 'versions': {}}
def cache_hash_package(self, category, name):
return '%s/%s' % (category, name)
@ -59,22 +60,28 @@ class Command(BaseCommand):
self._cache['packages'][key] = package
def cache_get_package(self, category, name):
return self._cache['packages'].get(self.cache_hash_package(category, name))
return self._cache['packages'].get(
self.cache_hash_package(category, name)
)
def cache_hash_version(self, category, name, version, revision, slot, overlay):
def cache_hash_version(self, category, name, version, revision, slot,
overlay):
key = '%s/%s-%s-r%s %s %s' % (category, name,
version, revision,
slot, overlay)
return key
def cache_get_version(self, category, name, version, revision, slot, overlay):
key = self.cache_hash_version(category, name, version, revision, slot, overlay)
def cache_get_version(self, category, name, version, revision, slot,
overlay):
key = self.cache_hash_version(category, name, version, revision, slot,
overlay)
return self._cache['versions'].get(key)
def cache_store_version(self, version):
key = self.cache_hash_version(version.package.category, version.package.name,
version.version, version.revision, version.slot,
version.overlay)
key = self.cache_hash_version(
version.package.category, version.package.name, version.version,
version.revision, version.slot, version.overlay
)
self._cache['versions'][key] = version
def handle(self, *args, **options):
@ -119,7 +126,8 @@ class Command(BaseCommand):
cmd = ['eix', '-!']
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
output = output.strip().strip('\n').split('\n')
overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
@ -139,9 +147,10 @@ class Command(BaseCommand):
env = os.environ
env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages', '-x']
if query:
cmd.extend(['--exact', query])
cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages',
'-x']
if query:
cmd.extend(['--exact', query])
if self.options['all']:
if not self.options['quiet']:
@ -151,7 +160,8 @@ class Command(BaseCommand):
if not self.options['quiet']:
self.stdout.write('done\n')
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
communicate()[0]
output = output.strip().strip('\n')
if len(output) == 0:
@ -166,17 +176,23 @@ class Command(BaseCommand):
else:
Package.objects.filter(name=query).delete()
else:
sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % query))
sys.stderr.write(
self.style.ERROR(
"Unknown package '%s'\n" % query
)
)
return
output = output.split('\n')
output = output.split('\n')
packages = {}
line_re = re.compile(r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$')
line_re = re.compile(
r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$'
)
package = None
for line in output:
for line in output:
match = line_re.match(line)
if not match:
@ -190,7 +206,8 @@ class Command(BaseCommand):
packages['%s/%s' % (cat, pkg)] = True
if not package or not (cat == package.category and pkg == package.name):
if not package or not \
(cat == package.category and pkg == package.name):
package = self.store_package(cat, pkg)
self.store_version(package, cpv, slot, overlay)
@ -208,16 +225,23 @@ class Command(BaseCommand):
obj = self.cache_get_package(cat, pkg)
if not obj:
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
obj, created = Package.objects.get_or_create(
category=cat,
name=pkg
)
self.cache_store_package(obj)
if created:
if not self.options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
' Set all versions dead, then set found versions alive and delete old versions '
# Set all versions dead, then set found versions alive and
# delete old versions
if not self.options['all']:
Version.objects.filter(package=obj, packaged=True).update(alive=False)
Version.objects.filter(
package=obj,
packaged=True
).update(alive=False)
return obj
@ -232,11 +256,15 @@ class Command(BaseCommand):
overlay = 'gentoo'
created = False
obj = self.cache_get_version(package.category, package.name, ver, rev, slot, overlay)
obj = self.cache_get_version(
package.category, package.name, ver, rev, slot, overlay
)
if not obj:
obj, created = Version.objects.get_or_create(package=package, slot=slot,
revision=rev, version=ver,
overlay=overlay)
obj, created = Version.objects.get_or_create(
package=package, slot=slot,
revision=rev, version=ver,
overlay=overlay
)
obj.alive = True
obj.packaged = True
@ -245,7 +273,8 @@ class Command(BaseCommand):
if created:
self.cache_store_version(obj)
''' nothing to do (note: it can't be an upstream version because overlay can't be empty here) '''
# nothing to do (note: it can't be an upstream version because
# overlay can't be empty here)
if not created:
return
@ -287,8 +316,10 @@ class Command(BaseCommand):
if self.options['no-log']:
continue
entry = VersionLog.objects.create(package=version.package,
action=VersionLog.VERSION_REMOVED)
entry = VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED
)
entry.slot = version.slot
entry.revision = version.revision
entry.version = version.version
@ -296,4 +327,3 @@ class Command(BaseCommand):
entry.save()
Version.objects.filter(packaged=True, alive=False).delete()

38
euscanwww/djeuscan/management/commands/scan-upstream.py
View File

@ -1,19 +1,17 @@
import subprocess
import portage
import sys
import os
import re
from StringIO import StringIO
from datetime import datetime
from portage import versions
from optparse import make_option
from django.utils import timezone
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from djeuscan.models import Package, Version, EuscanResult, VersionLog
class Command(BaseCommand):
_overlays = {}
@ -60,7 +58,7 @@ class Command(BaseCommand):
elif args:
packages = list(args)
else:
packages = [ package[:-1] for package in sys.stdin.readlines() ]
packages = [package[:-1] for package in sys.stdin.readlines()]
self.scan(options, packages)
@ -83,8 +81,12 @@ class Command(BaseCommand):
def parse_output(self, options, output):
from portage.versions import _cp
package_re = re.compile(r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$')
version_re = re.compile(r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$')
package_re = re.compile(
r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$'
)
version_re = re.compile(
r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$'
)
package = None
log = ""
@ -124,7 +126,6 @@ class Command(BaseCommand):
obj.datetime = timezone.now()
obj.save()
def store_package(self, options, cpv):
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
@ -133,15 +134,16 @@ class Command(BaseCommand):
if created and not options['quiet']:
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
' Set all versions dead, then set found versions alive and delete old versions '
# Set all versions dead, then set found versions alive and
# delete old versions
Version.objects.filter(package=obj, packaged=False).update(alive=False)
return obj
def store_version(self, options, package, ver, url):
obj, created = Version.objects.get_or_create(package=package, slot='',
revision='r0', version=ver,
overlay='')
obj, created = Version.objects.get_or_create(
package=package, slot='', revision='r0', version=ver, overlay=''
)
obj.alive = True
obj.urls = url
@ -155,7 +157,10 @@ class Command(BaseCommand):
if not options['quiet']:
sys.stdout.write('+ [u] %s %s\n' % (obj, url))
entry = VersionLog.objects.create(package=package, action=VersionLog.VERSION_ADDED)
entry = VersionLog.objects.create(
package=package,
action=VersionLog.VERSION_ADDED
)
entry.slot = ''
entry.revision = 'r0'
entry.version = ver
@ -165,12 +170,14 @@ class Command(BaseCommand):
package.n_versions += 1
package.save()
@commit_on_success
def purge_versions(self, options):
' For each dead versions '
for version in Version.objects.filter(packaged=False, alive=False):
entry = VersionLog.objects.create(package=version.package, action=VersionLog.VERSION_REMOVED)
entry = VersionLog.objects.create(
package=version.package,
action=VersionLog.VERSION_REMOVED
)
entry.slot = version.slot
entry.revision = version.revision
entry.version = version.version
@ -183,4 +190,3 @@ class Command(BaseCommand):
if not options['quiet']:
sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
Version.objects.filter(packaged=False, alive=False).delete()

55
euscanwww/djeuscan/management/commands/update-counters.py
View File

@ -1,10 +1,7 @@
import datetime
from optparse import make_option
from django.db.models import Count, Sum
from django.db.transaction import commit_on_success
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand
from django.utils import timezone
from djeuscan.models import Package, Herd, Maintainer, Version
@ -13,6 +10,7 @@ from djeuscan import charts
from distutils.version import StrictVersion, LooseVersion
def compare_versions(version1, version2):
try:
return cmp(StrictVersion(version1), StrictVersion(version2))
@ -20,6 +18,7 @@ def compare_versions(version1, version2):
except ValueError:
return cmp(LooseVersion(version1), LooseVersion(version2))
class Command(BaseCommand):
_overlays = {}
help = 'Update counters'
@ -98,7 +97,8 @@ class Command(BaseCommand):
return
if version['version'].startswith('9999'):
return
if compare_versions(storage[key]['version'], version['version']) < 0:
if compare_versions(storage[key]['version'],
version['version']) < 0:
storage[key] = version
if not options['fast']:
@ -125,25 +125,36 @@ class Command(BaseCommand):
package.n_packaged = n_packaged.get(package.id, 0)
package.n_overlay = n_overlay.get(package.id, 0)
default = {'id' : None}
package.last_version_gentoo_id = last_versions_gentoo.get(package.id, default)['id']
package.last_version_overlay_id = last_versions_overlay.get(package.id, default)['id']
package.last_version_upstream_id = last_versions_upstream.get(package.id, default)['id']
default = {'id': None}
package.last_version_gentoo_id = last_versions_gentoo.get(
package.id, default
)['id']
package.last_version_overlay_id = last_versions_overlay.get(
package.id, default
)['id']
package.last_version_upstream_id = last_versions_upstream.get(
package.id, default
)['id']
package.save()
n_packages_gentoo = int(package.n_packaged == package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged + package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay < package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged \
+ package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay \
< package.n_versions)
def update_row(storage, key):
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_outdated += n_packages_outdated
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
def update_log(storage, qs):
for row in qs:
update_row(storage, row['id'])
@ -153,13 +164,15 @@ class Command(BaseCommand):
update_log(maintainers, package.maintainers.all().values('id'))
update_row(categories, package.category)
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
if options['nolog']:
return

64
euscanwww/djeuscan/models.py
View File

@ -1,5 +1,6 @@
from django.db import models
class Herd(models.Model):
herd = models.CharField(max_length=128, unique=True)
email = models.CharField(max_length=128, blank=True, null=True)
@ -9,6 +10,7 @@ class Herd(models.Model):
return '%s <%s>' % (self.herd, self.email)
return self.herd
class Maintainer(models.Model):
name = models.CharField(max_length=128)
email = models.CharField(max_length=128, unique=True)
@ -16,6 +18,7 @@ class Maintainer(models.Model):
def __unicode__(self):
return '%s <%s>' % (self.name, self.email)
class Package(models.Model):
category = models.CharField(max_length=128)
name = models.CharField(max_length=128)
@ -30,15 +33,18 @@ class Package(models.Model):
n_overlay = models.IntegerField(default=0)
' And we also pre-compute last versions '
last_version_gentoo = models.ForeignKey('Version', blank=True, null=True,
related_name="last_version_gentoo",
on_delete=models.SET_NULL)
last_version_overlay = models.ForeignKey('Version', blank=True, null=True,
related_name="last_version_overlay",
on_delete=models.SET_NULL)
last_version_upstream = models.ForeignKey('Version', blank=True, null=True,
related_name="last_version_upstream",
on_delete=models.SET_NULL)
last_version_gentoo = models.ForeignKey(
'Version', blank=True, null=True, related_name="last_version_gentoo",
on_delete=models.SET_NULL
)
last_version_overlay = models.ForeignKey(
'Version', blank=True, null=True, related_name="last_version_overlay",
on_delete=models.SET_NULL
)
last_version_upstream = models.ForeignKey(
'Version', blank=True, null=True, related_name="last_version_upstream",
on_delete=models.SET_NULL
)
def __unicode__(self):
return '%s/%s' % (self.category, self.name)
@ -46,6 +52,7 @@ class Package(models.Model):
class Meta:
unique_together = ['category', 'name']
class Version(models.Model):
package = models.ForeignKey(Package)
slot = models.CharField(max_length=128)
@ -57,13 +64,15 @@ class Version(models.Model):
alive = models.BooleanField(default=True, db_index=True)
def __unicode__(self):
return '%s/%s-%s-%s:%s [%s]' % (self.package.category, self.package.name,
self.version, self.revision, self.slot,
self.overlay)
return '%s/%s-%s-%s:%s [%s]' % (
self.package.category, self.package.name, self.version,
self.revision, self.slot, self.overlay
)
class Meta:
unique_together = ['package', 'slot', 'revision', 'version', 'overlay']
class VersionLog(models.Model):
VERSION_ADDED = 1
VERSION_REMOVED = 2
@ -87,58 +96,67 @@ class VersionLog(models.Model):
def __unicode__(self):
txt = '+ ' if self.action == self.VERSION_ADDED else '- '
txt += '%s/%s-%s-%s:%s [%s]' % (self.package.category, self.package.name,
self.version, self.revision, self.slot,
self.overlay if self.overlay else '<upstream>')
txt += '%s/%s-%s-%s:%s [%s]' % (
self.package.category, self.package.name, self.version,
self.revision, self.slot,
self.overlay if self.overlay else '<upstream>'
)
return txt
class EuscanResult(models.Model):
package = models.ForeignKey(Package)
datetime = models.DateTimeField()
result = models.TextField(blank=True)
# Keep data for charts
class Log(models.Model):
datetime = models.DateTimeField()
' Packages up to date in the main portage tree '
n_packages_gentoo = models.IntegerField(default=0)
n_packages_gentoo = models.IntegerField(default=0)
' Packages up to date in an overlay '
n_packages_overlay = models.IntegerField(default=0)
n_packages_overlay = models.IntegerField(default=0)
' Packages outdated '
n_packages_outdated = models.IntegerField(default=0)
' Versions in the main portage tree '
n_versions_gentoo = models.IntegerField(default=0)
n_versions_gentoo = models.IntegerField(default=0)
' Versions in overlays '
n_versions_overlay = models.IntegerField(default=0)
n_versions_overlay = models.IntegerField(default=0)
' Upstream versions, not in the main tree or overlays '
n_versions_upstream = models.IntegerField(default=0)
def __unicode__(self):
return u'[%d:%d:%d] [%d:%d:%d]' % \
(self.n_packages_gentoo, self.n_packages_overlay, self.n_packages_outdated, \
self.n_versions_gentoo, self.n_versions_overlay, self.n_versions_upstream)
return u'[%d:%d:%d] [%d:%d:%d]' % (
self.n_packages_gentoo, self.n_packages_overlay,
self.n_packages_outdated, self.n_versions_gentoo,
self.n_versions_overlay, self.n_versions_upstream
)
class WorldLog(Log):
def __unicode__(self):
return u'world ' + Log.__unicode__(self)
class CategoryLog(Log):
category = models.CharField(max_length=128)
def __unicode__(self):
return u'%s %s' % (self.category, Log.__unicode__(self))
class HerdLog(Log):
herd = models.ForeignKey(Herd)
def __unicode__(self):
return u'%s %s' % (self.herd, Log.__unicode__(self))
class MaintainerLog(Log):
maintainer = models.ForeignKey(Maintainer)
def __unicode__(self):
return u'%s %s' % (self.maintainer, Log.__unicode__(self))

3
euscanwww/djeuscan/templatetags/div.py
View File

@ -2,7 +2,8 @@ from django import template
register = template.Library()
def div(value, arg=None):
return float(value)/float(arg)
return float(value) / float(arg)
register.filter('div', div)

14
euscanwww/djeuscan/templatetags/math_.py
View File

@ -1,11 +1,14 @@
from django.template import Node, Library
from django.template import Library
import math
register = Library()
# taken from http://lybniz2.sourceforge.net/safeeval.html
# make a list of safe functions
math_safe_list = ['acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'cosh', 'degrees', 'e', 'exp', 'fabs', 'floor', 'fmod', 'frexp', 'hypot', 'ldexp', 'log', 'log10', 'modf', 'pi', 'pow', 'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh']
math_safe_list = ['acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'cosh',
'degrees', 'e', 'exp', 'fabs', 'floor', 'fmod', 'frexp',
'hypot', 'ldexp', 'log', 'log10', 'modf', 'pi', 'pow',
'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh']
# use the list to filter the local namespace
math_safe_dict = dict([(k, getattr(math, k)) for k in math_safe_list])
@ -13,11 +16,13 @@ math_safe_dict = dict([(k, getattr(math, k)) for k in math_safe_list])
# add any needed builtins back in.
math_safe_dict['abs'] = abs
@register.filter('math')
def math_(lopr, expr):
"""Evals a math expression and returns it's value.
"$1" is a placeholder. Insert "$1" in the expression where the value is to be used. All math functions such as abs, sin, cos, floor are supported.
"$1" is a placeholder. Insert "$1" in the expression where the value is
to be used. All math functions such as abs, sin, cos, floor are supported.
Example,
a. You will be redirected in {{ seconds|math:"$1 / 60.0" }} minutes
b. Square of {{ x }} is {{ x|math:"$1 * $1" }}
@ -25,5 +30,6 @@ def math_(lopr, expr):
d. Given x = {{ x }}, (2 + x) * 6 = {{ x|math:"(2 + $1) * 6" }}
"""
if lopr:
return eval(expr.replace('$1', str(lopr)), {"__builtins__": None}, math_safe_dict)
return eval(expr.replace('$1', str(lopr)), {"__builtins__": None},
math_safe_dict)
return ''

3
euscanwww/djeuscan/templatetags/mul.py
View File

@ -2,7 +2,8 @@ from django import template
register = template.Library()
def mul(value, arg=None):
return value*arg
return value * arg
register.filter('mul', mul)

3
euscanwww/djeuscan/templatetags/packages.py
View File

@ -2,16 +2,19 @@ from django import template
register = template.Library()
@register.inclusion_tag('euscan/_packages.html', takes_context=True)
def packages(context, packages):
context['packages'] = packages
return context
@register.inclusion_tag('euscan/_package_cols.html', takes_context=True)
def package_cols(context, infos):
context['infos'] = infos
return context
@register.inclusion_tag('euscan/_package_bar.html', takes_context=True)
def package_bar(context, infos):
context['infos'] = infos

3
euscanwww/djeuscan/templatetags/sub.py
View File

@ -2,7 +2,8 @@ from django import template
register = template.Library()
def sub(value, arg=None):
return value-arg
return value - arg
register.filter('sub', sub)

7
euscanwww/djeuscan/templatetags/timedelta.py
View File

@ -6,6 +6,7 @@ from datetime import datetime
register = template.Library()
def timedelta(value, arg=None):
if not value:
return ''
@ -16,8 +17,8 @@ def timedelta(value, arg=None):
if settings.USE_TZ:
cmp = make_aware(cmp, get_default_timezone())
if value > cmp:
return "in %s" % timesince(cmp,value)
return "in %s" % timesince(cmp, value)
else:
return "%s ago" % timesince(value,cmp)
return "%s ago" % timesince(value, cmp)
register.filter('timedelta',timedelta)
register.filter('timedelta', timedelta)

2
euscanwww/djeuscan/tests.py
View File

@ -7,6 +7,7 @@ Replace these with more appropriate tests for your application.
from django.test import TestCase
class SimpleTest(TestCase):
def test_basic_addition(self):
"""
@ -20,4 +21,3 @@ Another way to test that 1 + 1 is equal to 2.
>>> 1 + 1 == 2
True
"""}

23
euscanwww/djeuscan/urls.py
View File

@ -1,29 +1,36 @@
from django.conf.urls.defaults import *
from feeds import *
from django.conf.urls.defaults import url, patterns, include
from feeds import PackageFeed, CategoryFeed, HerdFeed, MaintainerFeed, \
GlobalFeed
package_patterns = patterns('djeuscan.views',
url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/feed/$', PackageFeed(), name='package_feed'),
url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/feed/$',
PackageFeed(), name='package_feed'),
(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/$', 'package'),
)
categories_patterns = patterns('djeuscan.views',
(r'^(?P<category>[\w+][\w+.-]*)/(view/)?$', 'category'),
url(r'^(?P<category>[\w+][\w+.-]*)/feed/$', CategoryFeed(), name='category_feed'),
(r'^(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', 'chart_category'),
url(r'^(?P<category>[\w+][\w+.-]*)/feed/$', CategoryFeed(),
name='category_feed'),
(r'^(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$',
'chart_category'),
(r'^$', 'categories'),
)
herds_patterns = patterns('djeuscan.views',
(r'^(?P<herd>[\@\{\}\w+.-]*)/(view/)?$', 'herd'),
url(r'^(?P<herd>[\@\{\}\w+.-]*)/feed/$', HerdFeed(), name='herd_feed'),
(r'^(?P<herd>[\@\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', 'chart_herd'),
(r'^(?P<herd>[\@\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$',
'chart_herd'),
(r'^$', 'herds'),
)
maintainers_patterns = patterns('djeuscan.views',
(r'^(?P<maintainer_id>\d+)/(view/)?$', 'maintainer'),
url(r'^(?P<maintainer_id>\d+)/feed/$', MaintainerFeed(), name='maintainer_feed'),
(r'^(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$', 'chart_maintainer'),
url(r'^(?P<maintainer_id>\d+)/feed/$', MaintainerFeed(),
name='maintainer_feed'),
(r'^(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$',
'chart_maintainer'),
(r'^$', 'maintainers'),
)

117
euscanwww/djeuscan/views.py
View File

@ -1,5 +1,5 @@
from annoying.decorators import render_to
from django.http import HttpResponse, Http404
from django.http import Http404
from django.shortcuts import get_object_or_404
from django.db.models import Sum, Max
@ -10,84 +10,117 @@ import charts
""" Views """
@render_to('euscan/index.html')
def index(request):
ctx = {}
ctx['n_packaged'] = charts.xint(Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum'])
ctx['n_overlay'] = charts.xint(Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum'])
ctx['n_versions'] = charts.xint(Package.objects.aggregate(Sum('n_versions'))['n_versions__sum'])
ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - ctx['n_overlay']
ctx['n_packaged'] = charts.xint(
Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum']
)
ctx['n_overlay'] = charts.xint(
Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum']
)
ctx['n_versions'] = charts.xint(
Package.objects.aggregate(Sum('n_versions'))['n_versions__sum']
)
ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - \
ctx['n_overlay']
ctx['n_packages'] = Package.objects.count()
ctx['n_herds'] = Herd.objects.count()
ctx['n_maintainers'] = Maintainer.objects.count()
try:
ctx['last_scan'] = EuscanResult.objects.get(id=EuscanResult.objects.aggregate(Max('id'))['id__max']).datetime
ctx['last_scan'] = EuscanResult.objects.get(
id=EuscanResult.objects.aggregate(Max('id'))['id__max']
).datetime
except EuscanResult.DoesNotExist:
ctx['last_scan'] = None
return ctx
@render_to('euscan/logs.html')
def logs(request):
return {}
@render_to('euscan/categories.html')
def categories(request):
categories = Package.objects.values('category').annotate(n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
categories = Package.objects.values('category').annotate(
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')
)
return {'categories': categories}
return { 'categories' : categories }
@render_to('euscan/category.html')
def category(request, category):
packages = Package.objects.filter(category=category)
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
packages = packages.select_related(
'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
)
print dir(packages[0])
if not packages:
raise Http404
return { 'category' : category, 'packages' : packages }
return {'category': category, 'packages': packages}
@render_to('euscan/herds.html')
def herds(request):
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of 'INNER JOIN'
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of
# 'INNER JOIN'
herds = Package.objects.filter(herds__isnull=False)
herds = herds.values('herds__herd').annotate(n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
return { 'herds' : herds }
herds = herds.values('herds__herd').annotate(
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
return {'herds': herds}
@render_to('euscan/herd.html')
def herd(request, herd):
herd = get_object_or_404(Herd, herd=herd)
packages = Package.objects.filter(herds__id=herd.id)
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
return { 'herd' : herd, 'packages' : packages }
packages = packages.select_related(
'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
)
return {'herd': herd, 'packages': packages}
@render_to('euscan/maintainers.html')
def maintainers(request):
maintainers = Package.objects.filter(maintainers__isnull=False)
maintainers = maintainers.values('maintainers__id', 'maintainers__name', 'maintainers__email')
maintainers = maintainers.annotate(n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
maintainers = maintainers.values(
'maintainers__id', 'maintainers__name', 'maintainers__email'
)
maintainers = maintainers.annotate(
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')
)
return {'maintainers': maintainers}
return { 'maintainers' : maintainers }
@render_to('euscan/maintainer.html')
def maintainer(request, maintainer_id):
maintainer = get_object_or_404(Maintainer, id=maintainer_id)
packages = Package.objects.filter(maintainers__id=maintainer.id)
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
return { 'maintainer' : maintainer, 'packages' : packages }
packages = packages.select_related(
'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
)
return {'maintainer': maintainer, 'packages': packages}
@render_to('euscan/overlays.html')
def overlays(request):
overlays = Package.objects.values('version__overlay')
overlays = overlays.exclude(version__overlay='')
overlays = overlays.distinct()
return { 'overlays' : overlays }
return {'overlays': overlays}
@render_to('euscan/overlay.html')
def overlay(request, overlay):
@ -97,7 +130,8 @@ def overlay(request, overlay):
packages = packages.filter(version__overlay=overlay).distinct()
if not packages:
raise Http404
return { 'overlay' : overlay, 'packages' : packages }
return {'overlay': overlay, 'packages': packages}
@render_to('euscan/package.html')
def package(request, category, package):
@ -120,19 +154,23 @@ def package(request, category, package):
packaged = sorted(packaged, key=version_key)
upstream = sorted(upstream, key=version_key)
log = EuscanResult.objects.filter(package=package).order_by('-datetime')[:1]
log = EuscanResult.objects.filter(package=package).\
order_by('-datetime')[:1]
log = log[0] if log else None
vlog = VersionLog.objects.filter(package=package).order_by('-id')
return { 'package' : package, 'packaged' : packaged,
'upstream' : upstream, 'log' : log, 'vlog' : vlog }
return {'package': package, 'packaged': packaged,
'upstream': upstream, 'log': log, 'vlog': vlog}
@render_to('euscan/world.html')
def world(request):
world_form = WorldForm()
packages_form = PackagesForm()
return { 'world_form' : world_form,
'packages_form' : packages_form }
return {'world_form': world_form,
'packages_form': packages_form}
@render_to('euscan/world_scan.html')
def world_scan(request):
@ -157,28 +195,34 @@ def world_scan(request):
except:
pass
return { 'packages' : packages }
return {'packages': packages}
@render_to("euscan/about.html")
def about(request):
return {}
@render_to("euscan/api.html")
def api(request):
return {}
@render_to("euscan/statistics.html")
def statistics(request):
return {}
def chart(request, **kwargs):
from django.views.static import serve
chart = kwargs['chart'] if 'chart' in kwargs else None
if 'maintainer_id' in kwargs:
kwargs['maintainer'] = get_object_or_404(Maintainer, id=kwargs['maintainer_id'])
kwargs['maintainer'] = get_object_or_404(
Maintainer,
id=kwargs['maintainer_id']
)
if 'herd' in kwargs:
kwargs['herd'] = get_object_or_404(Herd, herd=kwargs['herd'])
@ -202,11 +246,14 @@ def chart(request, **kwargs):
return serve(request, path, document_root=charts.CHARTS_ROOT)
def chart_maintainer(request, **kwargs):
return chart(request, **kwargs)
def chart_herd(request, **kwargs):
return chart(request, **kwargs)
def chart_category(request, **kwargs):
return chart(request, **kwargs)

1
euscanwww/euscanwww/urls.py
View File

@ -22,4 +22,3 @@ if settings.DEBUG:
'document_root': os.path.join(settings.EUSCAN_ROOT, 'htdocs'),
}),
)

3
euscanwww/manage.py
View File

@ -1,5 +1,6 @@
#!/usr/bin/env python
import os, sys
import os
import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "euscanwww.settings")

5
euscanwww/scripts/regen-seq.py
View File

@ -1,4 +1,5 @@
import os, sys
import os
import sys
sys.path.insert(0, '../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'euscanwww.settings'
@ -19,6 +20,6 @@ try:
rel_name = seq_name.split("_id_seq")[0]
to_update.append((seq_name, rel_name,))
for row in to_update:
c.execute(r"SELECT setval('%s', max(id)) FROM %s"%row)
c.execute(r"SELECT setval('%s', max(id)) FROM %s" % row)
finally:
c.close()

70
euscanwww/templates/_base.html
View File

@ -15,7 +15,7 @@
<body>
<div id="header">
<a href="http://www.gentoo.org">
<img id="logo" src="{{ STATIC_URL }}img/gentoo_org.png" />
<img id="logo" src="{{ STATIC_URL }}img/gentoo_org.png" />
</a>
{% block header %}<h1>Ebuild Upstream Scanner (euscan)</h1>{% endblock %}
</div>
@ -24,44 +24,44 @@
</div>
<div id="menus">
{% block menus %}
<div class="menu">
<ul>
{% block menu %}
<li><a href="{% url djeuscan.views.index %}">Home</a></li>
<li><a href="{% url djeuscan.views.categories %}">Categories</a></li>
<li><a href="{% url djeuscan.views.herds %}">Herds</a></li>
<li><a href="{% url djeuscan.views.maintainers %}">Maintainers</a></li>
<li><a href="{% url djeuscan.views.overlays %}">Overlays</a></li>
<li><a href="{% url djeuscan.views.world %}">Scan World</a></li>
<li><a href="{% url djeuscan.views.statistics %}">Statistics</a></li>
<!--
<li>---</li>
<li><a href="#">Login</a></li>
<li><a href="#">Register</a></li>
-->
<li>---</li>
{% block menu_feed %}
<li>
<img src="{{ STATIC_URL }}img/feed.png" alt="feed" />
<a title="Global Feed" href="{% url global_feed %}">Global Feed</a>
</li>
{% endblock %}
<li>---</li>
<li><a href="{% url djeuscan.views.api %}">API</a></li>
<li><a href="{% url djeuscan.views.about %}">About</a></li>
{% endblock %}
</ul>
</div>
<div class="menu">
<ul>
{% block menu %}
<li><a href="{% url djeuscan.views.index %}">Home</a></li>
<li><a href="{% url djeuscan.views.categories %}">Categories</a></li>
<li><a href="{% url djeuscan.views.herds %}">Herds</a></li>
<li><a href="{% url djeuscan.views.maintainers %}">Maintainers</a></li>
<li><a href="{% url djeuscan.views.overlays %}">Overlays</a></li>
<li><a href="{% url djeuscan.views.world %}">Scan World</a></li>
<li><a href="{% url djeuscan.views.statistics %}">Statistics</a></li>
<!--
<li>---</li>
<li><a href="#">Login</a></li>
<li><a href="#">Register</a></li>
-->
<li>---</li>
{% block menu_feed %}
<li>
<img src="{{ STATIC_URL }}img/feed.png" alt="feed" />
<a title="Global Feed" href="{% url global_feed %}">Global Feed</a>
</li>
{% endblock %}
<li>---</li>
<li><a href="{% url djeuscan.views.api %}">API</a></li>
<li><a href="{% url djeuscan.views.about %}">About</a></li>
{% endblock %}
</ul>
</div>
{% endblock %}
</div>
<div id="footer">
<p>
Questions, Comments, Corrections ?
Email: corentin.chary at gmail.com<br />
Copyright (C) 2011 <strong>Corentin Chary</strong><br />
Original Gentoo artwork and logos copyright (C) Gentoo Foundation.<br />
Design inspired by (stolen from) gentoo.org and bugs.gentoo.org.<br />
<em>This site is not an official Gentoo website.</em>
Questions, Comments, Corrections ?
Email: corentin.chary at gmail.com<br />
Copyright (C) 2011 <strong>Corentin Chary</strong><br />
Original Gentoo artwork and logos copyright (C) Gentoo Foundation.<br />
Design inspired by (stolen from) gentoo.org and bugs.gentoo.org.<br />
<em>This site is not an official Gentoo website.</em>
</p>
</div>
</body>

2
euscanwww/templates/euscan/_packages.html
View File

@ -19,7 +19,7 @@
<tr>
<td>
<a href="{% url djeuscan.views.package package.category package.name %}">
{{ package.category }}/{{ package.name }}
{{ package.category }}/{{ package.name }}
</a>
{% package_bar package %}
</td>

2
euscanwww/templates/euscan/category.html
View File

@ -47,7 +47,7 @@
<dd><img src="{% url djeuscan.views.chart_category category 'versions-monthly-small' %}" /></dd>
<dt>Packages</dt>
<dd><img src="{% url djeuscan.views.chart_category category 'packages-monthly-small' %}" /></dd>
</dl>
</dl>
</div>
{% endblock %}

2
euscanwww/templates/euscan/herd.html
View File

@ -45,7 +45,7 @@
<dd><img src="{% url djeuscan.views.chart_herd herd.herd 'versions-monthly-small' %}" /></dd>
<dt>Packages</dt>
<dd><img src="{% url djeuscan.views.chart_herd herd.herd 'packages-monthly-small' %}" /></dd>
</dl>
</dl>
</div>
{% endblock %}

2
euscanwww/templates/euscan/maintainer.html
View File

@ -45,7 +45,7 @@
<dd><img src="{% url djeuscan.views.chart_maintainer maintainer.id 'versions-monthly-small' %}" /></dd>
<dt>Packages</dt>
<dd><img src="{% url djeuscan.views.chart_maintainer maintainer.id 'packages-monthly-small' %}" /></dd>
</dl>
</dl>
</div>
{% endblock %}

10
euscanwww/templates/euscan/maintainers.html
View File

@ -24,11 +24,11 @@
<tr>
<td>
<a href="{% url djeuscan.views.maintainer maintainer.maintainers__id %}">
{% if maintainer.maintainers__name != maintainer.maintainers__email %}
{{ maintainer.maintainers__name }} &lt;{{ maintainer.maintainers__email }}&gt;
{% else %}
{{ maintainer.maintainers__name }}
{% endif %}
{% if maintainer.maintainers__name != maintainer.maintainers__email %}
{{ maintainer.maintainers__name }} &lt;{{ maintainer.maintainers__email }}&gt;
{% else %}
{{ maintainer.maintainers__name }}
{% endif %}
</a>
{% package_bar maintainer %}
</td>

36
euscanwww/templates/euscan/package.html
View File

@ -51,7 +51,7 @@
{% for maintainer in package.maintainers.all %}
{% if maintainer.name != maintainer.email %}
<a href="{% url djeuscan.views.maintainer maintainer.id %}">
{{ maintainer.name }}
{{ maintainer.name }}
</a>
&lt;{{ maintainer.email }}&gt;
{% else %}
@ -68,12 +68,12 @@
<ul>
{% for version in packaged %}
<li id="{{ version.version }}-{{version.revision }}:{{ version.slot }}-[{{ version.overlay }}]">
{% if version.overlay == "gentoo" %}
<img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
{% else %}
<img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
{% endif %}
{{ version.version }}-{{ version.revision }} :{{ version.slot }} [{{ version.overlay }}]
{% if version.overlay == "gentoo" %}
<img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
{% else %}
<img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
{% endif %}
{{ version.version }}-{{ version.revision }} :{{ version.slot }} [{{ version.overlay }}]
</li>
{% endfor %}
</ul>
@ -85,8 +85,8 @@
<ul>
{% for version in upstream %}
<li>
<img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
{{ version.version }} - {{ version.urls }}
<img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
{{ version.version }} - {{ version.urls }}
</li>
{% endfor %}
</ul>
@ -101,14 +101,16 @@
{% else %}
<li class="removed">
{% endif %}
{% if version.overlay == "gentoo" %}
<img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
{% else %}{% if version.overlay %}
<img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
{% else %}
<img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
{% endif %}{% endif %}
{{ version }} - {{ version.datetime }}
{% if version.overlay == "gentoo" %}
<img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
{% elif version.overlay %}
<img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
{% else %}
<img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
{% endif %}
{{ version }} - {{ version.datetime }}
</li>
{% endfor %}
</ul>

31
pym/euscan/__init__.py
View File

@ -5,7 +5,6 @@
__version__ = "git"
import sys
from portage.output import EOutput
@ -19,15 +18,16 @@ CONFIG = {
'brute-force-false-watermark': 50,
'scan-dir': True,
'oneshot': True,
'user-agent' : 'escan (http://euscan.iksaif.net)',
'skip-robots-txt' : False,
'cache' : False
'user-agent': 'escan (http://euscan.iksaif.net)',
'skip-robots-txt': False,
'cache': False
}
output = EOutput(CONFIG['quiet'])
BLACKLIST_VERSIONS = [
# Compatibility package for running binaries linked against a pre gcc 3.4 libstdc++, won't be updated
# Compatibility package for running binaries linked against a
# pre gcc 3.4 libstdc++, won't be updated
'>=sys-libs/libstdc++-v3-3.4',
]
@ -39,21 +39,24 @@ BLACKLIST_PACKAGES = [
]
SCANDIR_BLACKLIST_URLS = [
'mirror://rubygems/(.*)', # Not browsable
'mirror://gentoo/(.*)' # Directory too big
'mirror://rubygems/(.*)', # Not browsable
'mirror://gentoo/(.*)' # Directory too big
]
BRUTEFORCE_BLACKLIST_PACKAGES = [
'net-zope/plonepopoll' # infinite loop any http://plone.org/products/plonepopoll/releases/*/plonepopoll-2-6-1.tgz link will work
# infinite loop any
# http://plone.org/products/plonepopoll/releases/*/plonepopoll-2-6-1.tgz
# link will work
'net-zope/plonepopoll'
]
BRUTEFORCE_BLACKLIST_URLS = [
'http://(.*)dockapps.org/download.php/id/(.*)', # infinite loop
'http://hydra.nixos.org/build/(.*)', # infinite loop
'http://www.rennings.net/gentoo/distfiles/(.*)', # Doesn't respect 404, infinite loop
'http://art.gnome.org/download/(.*)', # Doesn't respect 404, infinite loop
'http://barelysufficient.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop
'http://olemarkus.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop
'http://(.*)dockapps.org/download.php/id/(.*)', # infinite loop
'http://hydra.nixos.org/build/(.*)', # infinite loop
'http://www.rennings.net/gentoo/distfiles/(.*)', # Doesn't respect 404, infinite loop
'http://art.gnome.org/download/(.*)', # Doesn't respect 404, infinite loop
'http://barelysufficient.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop
'http://olemarkus.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop
]
ROBOTS_TXT_BLACKLIST_DOMAINS = [

5
pym/euscan/handlers/__init__.py
View File

@ -1,6 +1,7 @@
from euscan.handlers import generic, php, pypi, rubygem, kde, cpan
handlers = [ kde, php, pypi, rubygem, cpan, generic ]
handlers = [kde, php, pypi, rubygem, cpan, generic]
def find_best_handler(cpv, url):
for handler in handlers:
@ -8,12 +9,14 @@ def find_best_handler(cpv, url):
return handler
return None
def scan(cpv, url):
handler = find_best_handler(cpv, url)
if handler:
return handler.scan(cpv, url)
return []
def brute_force(cpv, url):
handler = find_best_handler(cpv, url)
if handler:

21
pym/euscan/handlers/cpan.py
View File

@ -8,9 +8,11 @@ import euscan
_cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*")
def can_handle(cpv, url):
return url.startswith('mirror://cpan/')
def guess_package(cp, url):
match = _cpan_package_name_re.search(url)
@ -27,6 +29,7 @@ def guess_package(cp, url):
return pkg
def gentoo_mangle_version(up_pv):
pv = ""
@ -39,17 +42,20 @@ def gentoo_mangle_version(up_pv):
c = up_pv[i]
pv += c
digits += int(c.isdigit())
if c == '.': digits = 0
if c == '.':
digits = 0
else:
pv = up_pv
return helpers.gentoo_mangle_version(pv)
def cpan_trim_version(pv):
pv = re.sub('^[a-zA-Z]+', '', pv)
pv = re.sub('[a-zA-Z]$', '', pv)
return pv
def cpan_mangle_version(pv):
pos = pv.find('.')
if pos < 0:
@ -59,6 +65,7 @@ def cpan_mangle_version(pv):
up_pv = cpan_trim_version(up_pv)
return up_pv
def cpan_vercmp(cp, a, b):
try:
return float(a) - float(b)
@ -68,6 +75,7 @@ def cpan_vercmp(cp, a, b):
else:
return 1
def scan(cpv, url):
cp, ver, rev = portage.pkgsplit(cpv)
pkg = guess_package(cp, url)
@ -107,15 +115,20 @@ def scan(cpv, url):
if helpers.version_filtered(cp, up_ver, up_pv, cpan_vercmp):
continue
url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % \
(version['cpanid'][0], version['cpanid'][0:1], version['cpanid'], version['archive'])
url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % (
version['cpanid'][0],
version['cpanid'][0:1],
version['cpanid'],
version['archive']
)
if url == orig_url:
continue
ret.append(( url, pv ))
ret.append((url, pv))
return ret
def brute_force(cpv, url):
return []

31
pym/euscan/handlers/generic.py
View File

@ -6,10 +6,12 @@ from BeautifulSoup import BeautifulSoup
import portage
from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS
from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \
BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS
from euscan import helpers
import euscan
def scan_html(data, url, pattern):
soup = BeautifulSoup(data)
results = []
@ -28,6 +30,7 @@ def scan_html(data, url, pattern):
return results
def scan_ftp(data, url, pattern):
buf = StringIO.StringIO(data)
results = []
@ -40,6 +43,7 @@ def scan_ftp(data, url, pattern):
return results
def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
if not steps:
return []
@ -91,6 +95,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
return versions
def scan(cpv, url):
for bu in SCANDIR_BLACKLIST_URLS:
if re.match(bu, url):
@ -107,13 +112,17 @@ def scan(cpv, url):
if ver not in resolved_url:
newver = helpers.version_change_end_sep(ver)
if newver and newver in resolved_url:
euscan.output.einfo("Version: using %s instead of %s" % (newver, ver))
euscan.output.einfo(
"Version: using %s instead of %s" % (newver, ver)
)
ver = newver
template = helpers.template_from_url(resolved_url, ver)
if '${' not in template:
euscan.output.einfo("Url doesn't seems to depend on version: %s not found in %s"
% (ver, resolved_url))
euscan.output.einfo(
"Url doesn't seems to depend on version: %s not found in %s" %
(ver, resolved_url)
)
return []
else:
euscan.output.einfo("Scanning: %s" % template)
@ -121,6 +130,7 @@ def scan(cpv, url):
steps = helpers.generate_scan_paths(template)
return scan_directory_recursive(cp, ver, rev, "", steps, url)
def brute_force(cpv, url):
cp, ver, rev = portage.pkgsplit(cpv)
@ -155,8 +165,9 @@ def brute_force(cpv, url):
template = helpers.template_from_url(url, ver)
if '${PV}' not in template:
euscan.output.einfo("Url doesn't seems to depend on full version: %s not found in %s"
% (ver, url))
euscan.output.einfo(
"Url doesn't seems to depend on full version: %s not found in %s" %
(ver, url))
return []
else:
euscan.output.einfo("Brute forcing: %s" % template)
@ -187,11 +198,14 @@ def brute_force(cpv, url):
result.append([url, version])
if len(result) > CONFIG['brute-force-false-watermark']:
euscan.output.einfo("Broken server detected ! Skipping brute force.")
euscan.output.einfo(
"Broken server detected ! Skipping brute force."
)
return []
if CONFIG["brute-force-recursive"]:
for v in helpers.gen_versions(list(components), CONFIG["brute-force"]):
for v in helpers.gen_versions(list(components),
CONFIG["brute-force"]):
if v not in versions and tuple(v) not in done:
versions.append(v)
@ -200,5 +214,6 @@ def brute_force(cpv, url):
return result
def can_handle(cpv, url):
return True

4
pym/euscan/handlers/kde.py
View File

@ -1,10 +1,12 @@
from euscan.handlers import generic
def can_handle(cpv, url):
if url.startswith('mirror://kde/'):
return True
return False
def clean_results(results):
ret = []
@ -15,6 +17,7 @@ def clean_results(results):
return ret
def scan(cpv, url):
results = generic.scan(cpv, url)
@ -24,6 +27,7 @@ def scan(cpv, url):
return clean_results(results)
def brute_force(cpv, url):
results = generic.brute_force(cpv, url)

6
pym/euscan/handlers/php.py
View File

@ -6,6 +6,7 @@ import xml.dom.minidom
from euscan import helpers
import euscan
def can_handle(cpv, url):
if url.startswith('http://pear.php.net/get/'):
return True
@ -13,6 +14,7 @@ def can_handle(cpv, url):
return True
return False
def guess_package_and_channel(cp, url):
match = re.search('http://(.*)/get/(.*)-(.*).tgz', url)
@ -24,6 +26,7 @@ def guess_package_and_channel(cp, url):
return pkg, host
def scan(cpv, url):
cp, ver, rev = portage.pkgsplit(cpv)
pkg, channel = guess_package_and_channel(cp, url)
@ -61,9 +64,10 @@ def scan(cpv, url):
if url == orig_url:
continue
ret.append(( url, pv ))
ret.append((url, pv))
return ret
def brute_force(cpv, url):
return []

10
pym/euscan/handlers/pypi.py
View File

@ -1,5 +1,4 @@
import xmlrpclib
import pprint
import re
import portage
@ -7,9 +6,11 @@ import portage
from euscan import helpers
import euscan
def can_handle(cpv, url):
return url.startswith('mirror://pypi/')
def guess_package(cp, url):
match = re.search('mirror://pypi/\w+/(.*)/.*', url)
if match:
@ -19,10 +20,10 @@ def guess_package(cp, url):
return pkg
def scan(cpv, url):
'http://wiki.python.org/moin/PyPiXmlRpc'
package = guess_package(cpv, url)
euscan.output.einfo("Using PyPi XMLRPC: " + package)
@ -44,10 +45,11 @@ def scan(cpv, url):
if helpers.version_filtered(cp, ver, pv):
continue
urls = client.release_urls(package, up_pv)
urls = " ".join([ infos['url'] for infos in urls ])
ret.append(( urls, pv ))
urls = " ".join([infos['url'] for infos in urls])
ret.append((urls, pv))
return ret
def brute_force(cpv, url):
return []

9
pym/euscan/handlers/rubygem.py
View File

@ -6,9 +6,11 @@ import urllib2
from euscan import helpers
import euscan
def can_handle(cpv, url):
return url.startswith('mirror://rubygems/')
def guess_gem(cpv, url):
match = re.search('mirror://rubygems/(.*).gem', url)
if match:
@ -23,12 +25,14 @@ def guess_gem(cpv, url):
return pkg
def scan(cpv, url):
'http://guides.rubygems.org/rubygems-org-api/#gemversion'
gem = guess_gem(cpv, url)
if not gem:
euscan.output.eerror("Can't guess gem name using %s and %s" % (cpv, url))
euscan.output.eerror("Can't guess gem name using %s and %s" % \
(cpv, url))
return []
url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
@ -61,9 +65,10 @@ def scan(cpv, url):
if helpers.version_filtered(cp, ver, pv):
continue
url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
ret.append(( url, pv ))
ret.append((url, pv))
return ret
def brute_force(cpv, url):
return []

45
pym/euscan/helpers.py
View File

@ -18,6 +18,7 @@ from portage import dep
from euscan import CONFIG, BLACKLIST_VERSIONS, ROBOTS_TXT_BLACKLIST_DOMAINS
import euscan
def htop_vercmp(a, b):
def fixver(v):
if v in ['0.11', '0.12', '0.13']:
@ -27,12 +28,13 @@ def htop_vercmp(a, b):
return simple_vercmp(fixver(a), fixver(b))
VERSION_CMP_PACKAGE_QUIRKS = {
'sys-process/htop' : htop_vercmp
'sys-process/htop': htop_vercmp
}
_v_end = '((-|_)(pre|p|beta|b|alpha|a|rc|r)\d*)'
_v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))'
# Stolen from g-pypi
def gentoo_mangle_version(up_pv):
"""Convert PV to MY_PV if needed
@ -146,6 +148,7 @@ def gentoo_mangle_version(up_pv):
return pv
def cast_int_components(version):
for i, obj in enumerate(version):
try:
@ -154,6 +157,7 @@ def cast_int_components(version):
pass
return version
def simple_vercmp(a, b):
if a == b:
return 0
@ -173,11 +177,13 @@ def simple_vercmp(a, b):
else:
return 1
def vercmp(package, a, b):
if package in VERSION_CMP_PACKAGE_QUIRKS:
return VERSION_CMP_PACKAGE_QUIRKS[package](a, b)
return simple_vercmp(a, b)
def version_is_nightly(a, b):
a = pkg_resources.parse_version(a)
b = pkg_resources.parse_version(b)
@ -188,6 +194,7 @@ def version_is_nightly(a, b):
return True
return False
def version_blacklisted(cp, version):
rule = None
cpv = '%s-%s' % (cp, version)
@ -205,6 +212,7 @@ def version_blacklisted(cp, version):
euscan.output.einfo("%s is blacklisted by rule %s" % (cpv, bv))
return rule is not None
def version_change_end_sep(version):
match = re.match('.*' + _v_end, version)
if not match:
@ -218,6 +226,7 @@ def version_change_end_sep(version):
return None
return version.replace(end, newend)
def version_filtered(cp, base, version, vercmp=vercmp):
if vercmp(cp, base, version) >= 0:
return True
@ -230,6 +239,7 @@ def version_filtered(cp, base, version, vercmp=vercmp):
return False
def generate_templates_vars(version):
ret = []
@ -246,6 +256,7 @@ def generate_templates_vars(version):
ret.reverse()
return ret
def template_from_url(url, version):
prefix, chunks = url.split('://')
chunks = chunks.split('/')
@ -261,6 +272,7 @@ def template_from_url(url, version):
return prefix + "://" + "/".join(chunks)
def url_from_template(url, version):
components = split_version(version)
@ -270,6 +282,7 @@ def url_from_template(url, version):
return url
# Stolen from distutils.LooseVersion
# Used for brute force to increment the version
def split_version(version):
@ -282,6 +295,7 @@ def split_version(version):
pass
return components
def join_version(components):
version = ""
for i in range(len(components)):
@ -292,6 +306,7 @@ def join_version(components):
version += "."
return version
def increment_version(components, level):
n = len(components)
@ -307,6 +322,7 @@ def increment_version(components, level):
return components
def gen_versions(components, level):
n = len(components)
depth = level
@ -325,6 +341,7 @@ def gen_versions(components, level):
return versions
def timeout_for_url(url):
if 'sourceforge' in url:
timeout = 15
@ -332,13 +349,16 @@ def timeout_for_url(url):
timeout = 5
return timeout
class HeadRequest(urllib2.Request):
def get_method(self):
return "HEAD"
""" RobotParser cache """
rpcache = {}
def urlallowed(url):
if CONFIG['skip-robots-txt']:
return True
@ -359,7 +379,7 @@ def urlallowed(url):
baseurl = '%s://%s' % (protocol, domain)
robotsurl = urlparse.urljoin(baseurl, 'robots.txt')
if rpcache.has_key(baseurl):
if baseurl in rpcache:
rp = rpcache[baseurl]
else:
from socket import setdefaulttimeout, getdefaulttimeout
@ -379,6 +399,7 @@ def urlallowed(url):
return rp.can_fetch(CONFIG['user-agent'], url) if rp else False
def urlopen(url, timeout=None, verb="GET"):
if not urlallowed(url):
euscan.output.einfo("Url '%s' blocked by robots.txt" % url)
@ -410,6 +431,7 @@ def urlopen(url, timeout=None, verb="GET"):
return opener.open(request, None, timeout)
def tryurl(fileurl, template):
result = True
@ -429,13 +451,16 @@ def tryurl(fileurl, template):
headers = fp.info()
if 'Content-disposition' in headers and basename not in headers['Content-disposition']:
if 'Content-disposition' in headers and \
basename not in headers['Content-disposition']:
result = None
elif 'Content-Length' in headers and headers['Content-Length'] == '0':
result = None
elif 'Content-Type' in headers and 'text/html' in headers['Content-Type']:
elif 'Content-Type' in headers and \
'text/html' in headers['Content-Type']:
result = None
elif 'Content-Type' in headers and 'application/x-httpd-php' in headers['Content-Type']:
elif 'Content-Type' in headers and \
'application/x-httpd-php' in headers['Content-Type']:
result = None
elif fp.geturl() != fileurl:
regex = regex_from_template(template)
@ -443,10 +468,10 @@ def tryurl(fileurl, template):
basename2 = os.path.basename(fp.geturl())
# Redirect to another (earlier?) version
if basename != basename2 and (re.match(regex, fp.geturl()) or re.match(baseregex, basename2)):
if basename != basename2 and (re.match(regex, fp.geturl()) or \
re.match(baseregex, basename2)):
result = None
if result:
result = (fp.geturl(), fp.info())
@ -459,6 +484,7 @@ def tryurl(fileurl, template):
return result
def regex_from_template(template):
# Escape
template = re.escape(template)
@ -483,6 +509,7 @@ def regex_from_template(template):
template = template + r'/?$'
return template
def basedir_from_template(template):
idx = template.find('${')
if idx == -1:
@ -494,6 +521,7 @@ def basedir_from_template(template):
return template[0:idx]
def generate_scan_paths(url):
prefix, chunks = url.split('://')
chunks = chunks.split('/')
@ -511,6 +539,7 @@ def generate_scan_paths(url):
return steps
def parse_mirror(uri):
from random import shuffle
@ -526,7 +555,7 @@ def parse_mirror(uri):
return None
mirrorname = uri[9:eidx]
path = uri[eidx+1:]
path = uri[eidx + 1:]
if mirrorname in mirrors:
mirrors = mirrors[mirrorname]

26
pym/euscan/scan.py
View File

@ -4,10 +4,8 @@ import sys
import portage
from portage.dbapi import porttree
from portage.output import white, yellow, turquoise, green, teal, red, EOutput
import gentoolkit.pprinter as pp
from gentoolkit import errors
from gentoolkit.query import Query
from gentoolkit.eclean.search import (port_settings)
@ -17,6 +15,7 @@ from euscan import helpers
import euscan
def filter_versions(cp, versions):
filtered = {}
@ -32,7 +31,8 @@ def filter_versions(cp, versions):
filtered[version] = url
return [ (cp, filtered[version], version) for version in filtered ]
return [(cp, filtered[version], version) for version in filtered]
def scan_upstream_urls(cpv, urls):
versions = []
@ -72,7 +72,9 @@ def scan_upstream(query):
)
if not matches:
sys.stderr.write(pp.warn("No package matching '%s'" % pp.pkgquery(query)))
sys.stderr.write(
pp.warn("No package matching '%s'" % pp.pkgquery(query))
)
return []
matches = sorted(matches)
@ -87,11 +89,15 @@ def scan_upstream(query):
return []
if pkg.cp in BLACKLIST_PACKAGES:
sys.stderr.write(pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp)))
sys.stderr.write(
pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp))
)
return []
if not CONFIG['quiet']:
pp.uprint(" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name())))
pp.uprint(
" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name()))
)
pp.uprint()
ebuild_path = pkg.ebuild_path()
@ -104,8 +110,8 @@ def scan_upstream(query):
cpv = pkg.cpv
metadata = {
"EAPI" : port_settings["EAPI"],
"SRC_URI" : pkg.environment("SRC_URI", False),
"EAPI": port_settings["EAPI"],
"SRC_URI": pkg.environment("SRC_URI", False),
}
use = frozenset(port_settings["PORTAGE_USE"].split())
try:
@ -113,7 +119,9 @@ def scan_upstream(query):
aalist = porttree._parse_uri_map(cpv, metadata)
except Exception as e:
sys.stderr.write(pp.warn("%s\n" % str(e)))
sys.stderr.write(pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv)))
sys.stderr.write(
pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv))
)
return []
if "mirror" in portage.settings.features:

4
setup.py
View File

@ -81,8 +81,8 @@ setup(
'https://github.com/iksaif/euscan/tarball/' +
('master' if __version__ == '9999' else ('euscan-%s' % __version__))
),
install_requires=['Django==1.3.1', 'django-annoying', 'South',
'django-piston', 'matplotlib'],
install_requires=['Django==1.4', 'django-annoying', 'South',
'django-piston', 'matplotlib', 'BeautifulSoup'],
package_dir={'': 'pym'},
packages=packages,
package_data={},