Merge pull request #2 from volpino/master
Cleaning up and updating requirements
This commit is contained in:
commit
1a40cf7bcc
9
README
9
README
@ -88,6 +88,15 @@ tree to avoid messing with your system.
|
||||
|
||||
### Installation
|
||||
|
||||
Install requirements from PyPI using
|
||||
|
||||
$ python setup.py develop
|
||||
|
||||
Extra dependencies:
|
||||
|
||||
* portage python api
|
||||
* rrdtool[python]
|
||||
|
||||
Like any django web app, just start by editing settings.py and then run
|
||||
these two commands.
|
||||
|
||||
|
28
bin/euscan
28
bin/euscan
@ -34,6 +34,7 @@ from euscan.scan import scan_upstream
|
||||
|
||||
""" Globals """
|
||||
|
||||
|
||||
def setupSignals():
|
||||
""" This block ensures that ^C interrupts are handled quietly. """
|
||||
import signal
|
||||
@ -68,7 +69,8 @@ def printUsage(_error=None, help=None):
|
||||
out = sys.stderr
|
||||
if not _error in ('global-options', 'packages',):
|
||||
_error = None
|
||||
if not _error and not help: help = 'all'
|
||||
if not _error and not help:
|
||||
help = 'all'
|
||||
if _error in ('global-options',):
|
||||
print(pp.error("Wrong option on command line."), file=out)
|
||||
print(file=out)
|
||||
@ -91,21 +93,24 @@ def printUsage(_error=None, help=None):
|
||||
" - turn off colors on output", file=out)
|
||||
print(yellow(" -q, --quiet") +
|
||||
" - be as quiet as possible", file=out)
|
||||
print( yellow(" -h, --help")+ \
|
||||
print(yellow(" -h, --help") +
|
||||
" - display the help screen", file=out)
|
||||
print(yellow(" -V, --version") +
|
||||
" - display version info", file=out)
|
||||
print(file=out)
|
||||
print(yellow(" -1, --oneshot") +
|
||||
" - stop as soon as a new version is found", file=out)
|
||||
" - stop as soon as a new version is found",
|
||||
file=out)
|
||||
print(yellow(" -b, --brute-force=<level>") +
|
||||
" - define the brute force "+yellow("<level>")+" (default: 2)\n" +
|
||||
" - define the brute force " + yellow("<level>") +
|
||||
" (default: 2)\n" +
|
||||
" " * 29 + "bigger levels will generate more versions numbers\n" +
|
||||
" " * 29 + "0 means disabled", file=out)
|
||||
print(file=out)
|
||||
if _error in ('packages',) or help:
|
||||
print(green(" package") +
|
||||
" - the packages (or ebuilds) you want to scan", file=out)
|
||||
" - the packages (or ebuilds) you want to scan",
|
||||
file=out)
|
||||
print(file=out)
|
||||
'''print( "More detailed instruction can be found in",
|
||||
turquoise("`man %s`" % __productname__), file=out)'''
|
||||
@ -115,6 +120,7 @@ class ParseArgsException(Exception):
|
||||
"""For parseArgs() -> main() communications."""
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.value)
|
||||
|
||||
@ -174,6 +180,7 @@ def parseArgs():
|
||||
|
||||
return args
|
||||
|
||||
|
||||
def main():
|
||||
"""Parse command line and execute all actions."""
|
||||
CONFIG['nocolor'] = (port_settings["NOCOLOR"] in ('yes', 'true')
|
||||
@ -215,8 +222,13 @@ def main():
|
||||
from os.path import basename # To get the short name
|
||||
|
||||
print(file=sys.stderr)
|
||||
print(pp.error("The short ebuild name '%s' is ambiguous. Please specify" % basename(pkgs[0])),
|
||||
file=sys.stderr, end="")
|
||||
print(
|
||||
pp.error(
|
||||
"The short ebuild name '%s' is ambiguous. Please specify" \
|
||||
% basename(pkgs[0])
|
||||
),
|
||||
file=sys.stderr, end=""
|
||||
)
|
||||
pp.die(1, "one of the above fully-qualified ebuild names instead.")
|
||||
except GentoolkitException as err:
|
||||
pp.die(1, '%s: %s' % (package, str(err)))
|
||||
@ -238,7 +250,7 @@ def main():
|
||||
if not len(ret) and not CONFIG['quiet']:
|
||||
print (pp.warn("Didn't find any new version, "
|
||||
+ "check package's homepage for "
|
||||
+ "more informations"));
|
||||
+ "more informations"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
@ -1,11 +1,14 @@
|
||||
from models import *
|
||||
from models import Package, Version, VersionLog, EuscanResult, Log, WorldLog, \
|
||||
CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer
|
||||
from django.contrib import admin
|
||||
|
||||
admin.site.register(Herd)
|
||||
admin.site.register(Maintainer)
|
||||
|
||||
|
||||
class PackageAdmin(admin.ModelAdmin):
|
||||
search_fields = ('category', 'name')
|
||||
|
||||
admin.site.register(Package, PackageAdmin)
|
||||
|
||||
admin.site.register(Version)
|
||||
|
@ -1,4 +1,5 @@
|
||||
from piston.emitters import Emitter, XMLEmitter
|
||||
from piston.emitters import XMLEmitter
|
||||
|
||||
|
||||
class EuscanXMLEmitter(XMLEmitter):
|
||||
_parent = []
|
||||
|
@ -1,24 +1,26 @@
|
||||
from piston.handler import AnonymousBaseHandler, BaseHandler
|
||||
from piston.utils import rc, HttpStatusCode
|
||||
from piston.handler import AnonymousBaseHandler
|
||||
from piston.utils import rc
|
||||
|
||||
from django.db.models import Sum, Max
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
from django.forms.models import model_to_dict
|
||||
|
||||
from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, VersionLog
|
||||
from djeuscan.forms import WorldForm, PackagesForm
|
||||
from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
|
||||
VersionLog
|
||||
|
||||
# replace default XMLEmitter with ours
|
||||
from piston.emitters import Emitter
|
||||
from emitters import EuscanXMLEmitter
|
||||
Emitter.register('xml', EuscanXMLEmitter, 'text/xml; charset=utf-8')
|
||||
|
||||
|
||||
def xint(i):
|
||||
try:
|
||||
return int(i)
|
||||
except:
|
||||
return 0
|
||||
|
||||
|
||||
def renameFields(vqs, fields):
|
||||
ret = []
|
||||
for n in vqs:
|
||||
@ -29,6 +31,7 @@ def renameFields(vqs, fields):
|
||||
ret.append(n)
|
||||
return ret
|
||||
|
||||
|
||||
class catch_and_return(object):
|
||||
def __init__(self, err, response):
|
||||
self.err = err
|
||||
@ -42,6 +45,7 @@ class catch_and_return(object):
|
||||
return self.response
|
||||
return wrapper
|
||||
|
||||
|
||||
# /api/1.0/
|
||||
class RootHandler(AnonymousBaseHandler):
|
||||
allowed_methods = ('GET',)
|
||||
@ -49,53 +53,71 @@ class RootHandler(AnonymousBaseHandler):
|
||||
def read(self, request):
|
||||
return {'api-version': '1.0'}
|
||||
|
||||
|
||||
# /api/1.0/statistics
|
||||
class StatisticsHandler(AnonymousBaseHandler):
|
||||
allowed_methods = ('GET',)
|
||||
|
||||
def read(self, request):
|
||||
_aggr = Package.objects.aggregate
|
||||
data = {}
|
||||
data['n_packaged'] = xint(Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum'])
|
||||
data['n_overlay'] = xint(Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum'])
|
||||
data['n_versions'] = xint(Package.objects.aggregate(Sum('n_versions'))['n_versions__sum'])
|
||||
data['n_upstream'] = data['n_versions'] - data['n_packaged'] - data['n_overlay']
|
||||
data['n_packaged'] = xint(_aggr(Sum('n_packaged'))['n_packaged__sum'])
|
||||
data['n_overlay'] = xint(_aggr(Sum('n_overlay'))['n_overlay__sum'])
|
||||
data['n_versions'] = xint(_aggr(Sum('n_versions'))['n_versions__sum'])
|
||||
data['n_upstream'] = data['n_versions'] - data['n_packaged'] - \
|
||||
data['n_overlay']
|
||||
data['n_packages'] = Package.objects.count()
|
||||
data['n_herds'] = Herd.objects.count()
|
||||
data['n_maintainers'] = Maintainer.objects.count()
|
||||
data['last_scan'] = EuscanResult.objects.get(id=EuscanResult.objects.aggregate(Max('id'))['id__max']).datetime
|
||||
data['last_scan'] = EuscanResult.objects.get(
|
||||
id=EuscanResult.objects.aggregate(Max('id'))['id__max']
|
||||
).datetime
|
||||
|
||||
return data
|
||||
|
||||
|
||||
# /api/1.0/maintainers
|
||||
class MaintainersHandler(AnonymousBaseHandler):
|
||||
allowed_methods = ('GET',)
|
||||
|
||||
def read(self, request):
|
||||
maintainers = Package.objects.filter(maintainers__isnull=False)
|
||||
maintainers = maintainers.values('maintainers__id', 'maintainers__name', 'maintainers__email')
|
||||
maintainers = maintainers.values(
|
||||
'maintainers__id',
|
||||
'maintainers__name',
|
||||
'maintainers__email'
|
||||
)
|
||||
maintainers = maintainers.annotate(n_packaged=Sum('n_packaged'),
|
||||
n_overlay=Sum('n_overlay'),
|
||||
n_versions=Sum('n_versions'))
|
||||
|
||||
maintainers = renameFields(maintainers, [('maintainers__id', 'id'),
|
||||
maintainers = renameFields(
|
||||
maintainers,
|
||||
[('maintainers__id', 'id'),
|
||||
('maintainers__name', 'name'),
|
||||
('maintainers__email', 'email')])
|
||||
('maintainers__email', 'email')]
|
||||
)
|
||||
return {'maintainers': maintainers}
|
||||
|
||||
|
||||
# /api/1.0/herds
|
||||
class HerdsHandler(AnonymousBaseHandler):
|
||||
allowed_methods = ('GET',)
|
||||
|
||||
def read(self, request):
|
||||
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of 'INNER JOIN'
|
||||
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN'
|
||||
# instead of 'INNER JOIN'
|
||||
herds = Package.objects.filter(herds__isnull=False)
|
||||
herds = herds.values('herds__herd').annotate(n_packaged=Sum('n_packaged'),
|
||||
herds = herds.values('herds__herd').annotate(
|
||||
n_packaged=Sum('n_packaged'),
|
||||
n_overlay=Sum('n_overlay'),
|
||||
n_versions=Sum('n_versions'))
|
||||
n_versions=Sum('n_versions')
|
||||
)
|
||||
|
||||
herds = renameFields(herds, [('herds__herd', 'herd')])
|
||||
return {'herds': herds}
|
||||
|
||||
|
||||
# /api/1.0/categories
|
||||
class CategoriesHandler(AnonymousBaseHandler):
|
||||
allowed_methods = ('GET',)
|
||||
@ -108,6 +130,7 @@ class CategoriesHandler(AnonymousBaseHandler):
|
||||
|
||||
return {'categories': categories}
|
||||
|
||||
|
||||
# /api/1.0/packages/by-maintainer/
|
||||
# /api/1.0/packages/by-category/
|
||||
# /api/1.0/packages/by-herd/
|
||||
@ -135,7 +158,11 @@ class PackagesHandler(AnonymousBaseHandler):
|
||||
packages = Package.objects.filter(maintainers__id=maintainer.id)
|
||||
data = {'maintainer': maintainer}
|
||||
|
||||
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
|
||||
packages = packages.select_related(
|
||||
'last_version_gentoo',
|
||||
'last_version_overlay',
|
||||
'last_version_upstream'
|
||||
)
|
||||
data['packages'] = packages
|
||||
|
||||
if not data:
|
||||
@ -143,6 +170,7 @@ class PackagesHandler(AnonymousBaseHandler):
|
||||
|
||||
return data
|
||||
|
||||
|
||||
# /api/1.0/package/
|
||||
class PackageHandler(AnonymousBaseHandler):
|
||||
allowed_methods = ('GET',)
|
||||
@ -152,7 +180,8 @@ class PackageHandler(AnonymousBaseHandler):
|
||||
package = Package.objects.get(category=category, name=package)
|
||||
package.homepages = package.homepage.split(' ')
|
||||
versions = Version.objects.filter(package=package)
|
||||
log = EuscanResult.objects.filter(package=package).order_by('-datetime')[:1]
|
||||
log = EuscanResult.objects.filter(package=package).\
|
||||
order_by('-datetime')[:1]
|
||||
log = log[0] if log else None
|
||||
vlog = VersionLog.objects.filter(package=package).order_by('-id')
|
||||
|
||||
@ -166,7 +195,8 @@ class PackageHandler(AnonymousBaseHandler):
|
||||
|
||||
version_log = []
|
||||
for v in vlog:
|
||||
v = model_to_dict(v, ['version', 'revision', 'slot', 'overlay', 'datetime', 'action'])
|
||||
v = model_to_dict(v, ['version', 'revision', 'slot', 'overlay',
|
||||
'datetime', 'action'])
|
||||
if v['action'] == VersionLog.VERSION_ADDED:
|
||||
v['action'] = 'added'
|
||||
if v['action'] == VersionLog.VERSION_REMOVED:
|
||||
@ -177,7 +207,10 @@ class PackageHandler(AnonymousBaseHandler):
|
||||
packaged = []
|
||||
for version in versions:
|
||||
unpackaged = not version.packaged
|
||||
version = model_to_dict(version, ['version', 'revision', 'slot', 'overlay', 'urls'])
|
||||
version = model_to_dict(
|
||||
version,
|
||||
['version', 'revision', 'slot', 'overlay', 'urls']
|
||||
)
|
||||
if unpackaged:
|
||||
upstream.append(version)
|
||||
else:
|
||||
|
@ -1,7 +1,8 @@
|
||||
from django.conf.urls.defaults import *
|
||||
from django.conf.urls.defaults import url, patterns
|
||||
|
||||
from piston.resource import Resource
|
||||
from handlers import *
|
||||
from handlers import RootHandler, StatisticsHandler, HerdsHandler, \
|
||||
CategoriesHandler, MaintainersHandler, PackagesHandler, PackageHandler
|
||||
|
||||
root_handler = Resource(handler=RootHandler)
|
||||
statistics_handler = Resource(handler=StatisticsHandler)
|
||||
@ -12,16 +13,25 @@ packages_handler = Resource(handler=PackagesHandler)
|
||||
package_handler = Resource(handler=PackageHandler)
|
||||
|
||||
urlpatterns = patterns('djeuscan.api.views',
|
||||
url(r'^1.0/statistics\.(?P<emitter_format>.+)$', statistics_handler, name='api.views.statistics'),
|
||||
url(r'^1.0/herds\.(?P<emitter_format>.+)$', herds_handler, name='api.views.herds'),
|
||||
url(r'^1.0/categories\.(?P<emitter_format>.+)$', categories_handler, name='api.views.categories'),
|
||||
url(r'^1.0/maintainers\.(?P<emitter_format>.+)$', maintainers_handler, name='api.views.maintainers'),
|
||||
url(r'^1.0/statistics\.(?P<emitter_format>.+)$', statistics_handler,
|
||||
name='api.views.statistics'),
|
||||
url(r'^1.0/herds\.(?P<emitter_format>.+)$', herds_handler,
|
||||
name='api.views.herds'),
|
||||
url(r'^1.0/categories\.(?P<emitter_format>.+)$', categories_handler,
|
||||
name='api.views.categories'),
|
||||
url(r'^1.0/maintainers\.(?P<emitter_format>.+)$', maintainers_handler,
|
||||
name='api.views.maintainers'),
|
||||
|
||||
url(r'^1.0/packages/by-maintainer/(?P<maintainer_id>\d+)\.(?P<emitter_format>.+)$', packages_handler, name='api.views.packages'),
|
||||
url(r'^1.0/packages/by-herd/(?P<herd>[\@\{\}\w+.-]*)\.(?P<emitter_format>.+)?$', packages_handler, name='api.views.packages'),
|
||||
url(r'^1.0/packages/by-category/(?P<category>[\w+][\w+.-]*)\.(?P<emitter_format>.+)?$', packages_handler, name='api.views.packages'),
|
||||
url(r'^1.0/packages/by-maintainer/(?P<maintainer_id>\d+)\.(?P<emitter_format>.+)$',
|
||||
packages_handler, name='api.views.packages'),
|
||||
url(r'^1.0/packages/by-herd/(?P<herd>[\@\{\}\w+.-]*)\.(?P<emitter_format>.+)?$',
|
||||
packages_handler, name='api.views.packages'),
|
||||
url(r'^1.0/packages/by-category/(?P<category>[\w+][\w+.-]*)\.(?P<emitter_format>.+)?$',
|
||||
packages_handler, name='api.views.packages'),
|
||||
|
||||
url(r'^1.0/package/(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)\.(?P<emitter_format>.+)$', package_handler, name='api.views.package'),
|
||||
url(r'^1.0/package/(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)\.(?P<emitter_format>.+)$',
|
||||
package_handler, name='api.views.package'),
|
||||
|
||||
url(r'^1.0/api\.(?P<emitter_format>.+)$', root_handler, name='api.views.root'),
|
||||
url(r'^1.0/api\.(?P<emitter_format>.+)$',
|
||||
root_handler, name='api.views.root'),
|
||||
)
|
||||
|
@ -3,14 +3,12 @@ import time
|
||||
|
||||
from euscanwww import settings
|
||||
|
||||
from django.db.models import F, Sum, Max
|
||||
from djeuscan.models import Version, Package, Herd, Maintainer
|
||||
from djeuscan.models import CategoryLog
|
||||
from django.db.models import F, Sum
|
||||
from djeuscan.models import Package
|
||||
|
||||
import rrdtool
|
||||
|
||||
import pylab
|
||||
import matplotlib
|
||||
|
||||
CHARTS_ROOT = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts')
|
||||
CHARTS_URL = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts')
|
||||
@ -21,12 +19,14 @@ pylab.rcParams['axes.titlesize'] = 10.0
|
||||
pylab.rcParams['xtick.labelsize'] = 8.0
|
||||
pylab.rcParams['legend.fontsize'] = 8.0
|
||||
|
||||
|
||||
def xint(i):
|
||||
try:
|
||||
return int(i)
|
||||
except:
|
||||
return 0
|
||||
|
||||
|
||||
def chart_alive(name):
|
||||
path = os.path.join(CHARTS_ROOT, name)
|
||||
if not os.path.exists(path):
|
||||
@ -37,6 +37,7 @@ def chart_alive(name):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def rrd_name(**kwargs):
|
||||
name = ""
|
||||
|
||||
@ -51,6 +52,7 @@ def rrd_name(**kwargs):
|
||||
|
||||
return name
|
||||
|
||||
|
||||
def chart_name(name, **kwargs):
|
||||
name = name.replace('_', '-')
|
||||
|
||||
@ -67,6 +69,7 @@ def chart_name(name, **kwargs):
|
||||
|
||||
return name + ".png"
|
||||
|
||||
|
||||
def getpackages(**kwargs):
|
||||
packages = Package.objects
|
||||
|
||||
@ -79,6 +82,7 @@ def getpackages(**kwargs):
|
||||
|
||||
return packages
|
||||
|
||||
|
||||
def cached_pylab_chart(f):
|
||||
def new_f(*args, **kwds):
|
||||
name = chart_name(f.func_name, **kwds)
|
||||
@ -93,11 +97,13 @@ def cached_pylab_chart(f):
|
||||
new_f.func_name = f.func_name
|
||||
return new_f
|
||||
|
||||
|
||||
@cached_pylab_chart
|
||||
def pie_versions(**kwargs):
|
||||
n_packaged = xint(getpackages(**kwargs).aggregate(Sum('n_packaged'))['n_packaged__sum'])
|
||||
n_overlay = xint(getpackages(**kwargs).aggregate(Sum('n_overlay'))['n_overlay__sum'])
|
||||
n_versions = xint(getpackages(**kwargs).aggregate(Sum('n_versions'))['n_versions__sum'])
|
||||
gpk = getpackages(**kwargs)
|
||||
n_packaged = xint(gpk.aggregate(Sum('n_packaged'))['n_packaged__sum'])
|
||||
n_overlay = xint(gpk.aggregate(Sum('n_overlay'))['n_overlay__sum'])
|
||||
n_versions = xint(gpk.aggregate(Sum('n_versions'))['n_versions__sum'])
|
||||
n_upstream = n_versions - n_packaged - n_overlay
|
||||
|
||||
pylab.figure(1, figsize=(3.5, 3.5))
|
||||
@ -111,35 +117,43 @@ def pie_versions(**kwargs):
|
||||
fracs = [n_packaged, n_upstream]
|
||||
colors = '#008000', '#FF0000'
|
||||
|
||||
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', shadow=True)
|
||||
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%',
|
||||
shadow=True)
|
||||
pylab.title('Versions', bbox={'facecolor': '0.8', 'pad': 5})
|
||||
|
||||
|
||||
@cached_pylab_chart
|
||||
def pie_packages(**kwargs):
|
||||
n_packages = getpackages(**kwargs).count()
|
||||
n_packages_uptodate_main = getpackages(**kwargs).filter(n_versions=F('n_packaged')).count()
|
||||
n_packages_uptodate_all = getpackages(**kwargs).filter(n_versions=F('n_packaged') + F('n_overlay')).count()
|
||||
gpk = getpackages(**kwargs)
|
||||
n_packages = gpk.count()
|
||||
n_packages_uptodate_main = gpk.filter(n_versions=F('n_packaged')).count()
|
||||
n_packages_uptodate_all = gpk.filter(n_versions=F('n_packaged') + \
|
||||
F('n_overlay')).count()
|
||||
n_packages_outdated = n_packages - n_packages_uptodate_all
|
||||
n_packages_uptodate_ovl = n_packages_uptodate_all - n_packages_uptodate_main
|
||||
n_packages_uptodate_ovl = n_packages_uptodate_all - \
|
||||
n_packages_uptodate_main
|
||||
|
||||
pylab.figure(1, figsize=(3.5, 3.5))
|
||||
|
||||
if n_packages_uptodate_ovl:
|
||||
labels = 'Ok (gentoo)', 'Ok (overlays)', 'Outdated'
|
||||
fracs = [n_packages_uptodate_main, n_packages_uptodate_ovl, n_packages_outdated]
|
||||
fracs = [n_packages_uptodate_main, n_packages_uptodate_ovl,
|
||||
n_packages_outdated]
|
||||
colors = '#008000', '#0B17FD', '#FF0000'
|
||||
else:
|
||||
labels = 'Ok (gentoo)', 'Outdated'
|
||||
fracs = [n_packages_uptodate_main, n_packages_outdated]
|
||||
colors = '#008000', '#FF0000'
|
||||
|
||||
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%', shadow=True)
|
||||
pylab.pie(fracs, labels=labels, colors=colors, autopct='%1.1f%%',
|
||||
shadow=True)
|
||||
pylab.title('Packages', bbox={'facecolor': '0.8', 'pad': 5})
|
||||
|
||||
|
||||
def rrd_path(name):
|
||||
return str(os.path.join(settings.RRD_ROOT, name + '.rrd'))
|
||||
|
||||
|
||||
def rrd_create(name, start):
|
||||
path = rrd_path(name)
|
||||
if os.path.exists(path):
|
||||
@ -156,13 +170,17 @@ def rrd_create(name, start):
|
||||
'RRA:AVERAGE:0.5:5:200',
|
||||
'RRA:AVERAGE:0.5:10:200')
|
||||
|
||||
|
||||
def rrd_update(name, datetime, values):
|
||||
now = time.mktime(datetime.timetuple())
|
||||
rrd_create(name, now)
|
||||
rrdtool.update(rrd_path(name),
|
||||
rrdtool.update(
|
||||
rrd_path(name),
|
||||
'%d:%d:%d:%d:%d:%d:%d' % \
|
||||
(now, values.n_packages_gentoo, values.n_packages_overlay, values.n_packages_outdated, \
|
||||
values.n_versions_gentoo, values.n_versions_overlay, values.n_versions_upstream))
|
||||
(now, values.n_packages_gentoo, values.n_packages_overlay,
|
||||
values.n_packages_outdated, values.n_versions_gentoo,
|
||||
values.n_versions_overlay, values.n_versions_upstream)
|
||||
)
|
||||
|
||||
|
||||
"""
|
||||
@ -175,6 +193,7 @@ def rrd_update(name, datetime, values):
|
||||
[-J|--alt-autoscale-min]
|
||||
"""
|
||||
|
||||
|
||||
def cached_rrd_chart(f):
|
||||
def new_f(*args, **kwds):
|
||||
if 'period' not in kwds:
|
||||
@ -215,9 +234,11 @@ def cached_rrd_chart(f):
|
||||
new_f.func_name = f.func_name
|
||||
return new_f
|
||||
|
||||
|
||||
@cached_rrd_chart
|
||||
def packages(**kwargs):
|
||||
rrdtool.graph(str(kwargs['path']),
|
||||
rrdtool.graph(
|
||||
str(kwargs['path']),
|
||||
'--imgformat', 'PNG',
|
||||
'--width', kwargs['width'],
|
||||
'--height', kwargs['height'],
|
||||
@ -231,15 +252,20 @@ def packages(**kwargs):
|
||||
'--title', kwargs['title'],
|
||||
'--lower-limit', '0',
|
||||
'DEF:n_packages_gentoo=%s:n_packages_gentoo:AVERAGE' % (kwargs['rrd']),
|
||||
'DEF:n_packages_overlay=%s:n_packages_overlay:AVERAGE' % (kwargs['rrd']),
|
||||
'DEF:n_packages_outdated=%s:n_packages_outdated:AVERAGE' % (kwargs['rrd']),
|
||||
'DEF:n_packages_overlay=%s:n_packages_overlay:AVERAGE' % \
|
||||
(kwargs['rrd']),
|
||||
'DEF:n_packages_outdated=%s:n_packages_outdated:AVERAGE' % \
|
||||
(kwargs['rrd']),
|
||||
'LINE1.25:n_packages_gentoo#008000:Gentoo',
|
||||
'LINE1.25:n_packages_overlay#0B17FD:Overlay',
|
||||
'LINE1.25:n_packages_outdated#FF0000:Outdated')
|
||||
'LINE1.25:n_packages_outdated#FF0000:Outdated'
|
||||
)
|
||||
|
||||
|
||||
@cached_rrd_chart
|
||||
def versions(**kwargs):
|
||||
rrdtool.graph(str(kwargs['path']),
|
||||
rrdtool.graph(
|
||||
str(kwargs['path']),
|
||||
'--imgformat', 'PNG',
|
||||
'--width', kwargs['width'],
|
||||
'--height', kwargs['height'],
|
||||
@ -252,10 +278,11 @@ def versions(**kwargs):
|
||||
'--title', kwargs['title'],
|
||||
'--lower-limit', '0',
|
||||
'DEF:n_versions_gentoo=%s:n_versions_gentoo:AVERAGE' % (kwargs['rrd']),
|
||||
'DEF:n_versions_overlay=%s:n_versions_overlay:AVERAGE' % (kwargs['rrd']),
|
||||
'DEF:n_versions_outdated=%s:n_versions_upstream:AVERAGE' % (kwargs['rrd']),
|
||||
'DEF:n_versions_overlay=%s:n_versions_overlay:AVERAGE' % \
|
||||
(kwargs['rrd']),
|
||||
'DEF:n_versions_outdated=%s:n_versions_upstream:AVERAGE' % \
|
||||
(kwargs['rrd']),
|
||||
'LINE1.25:n_versions_gentoo#008000:Gentoo',
|
||||
'LINE1.25:n_versions_overlay#0B17FD:Overlay',
|
||||
'LINE1.25:n_versions_outdated#FF0000:Outdated')
|
||||
|
||||
|
||||
'LINE1.25:n_versions_outdated#FF0000:Outdated'
|
||||
)
|
||||
|
@ -1,11 +1,10 @@
|
||||
from django.contrib.syndication.views import Feed, FeedDoesNotExist
|
||||
from django.shortcuts import get_object_or_404
|
||||
from django.http import Http404
|
||||
from django.utils.feedgenerator import Atom1Feed
|
||||
from django.core.urlresolvers import reverse
|
||||
|
||||
from djeuscan.models import Version, Package, Herd, Maintainer, VersionLog
|
||||
from djeuscan.views import *
|
||||
from djeuscan.models import Package, Herd, Maintainer, VersionLog
|
||||
|
||||
|
||||
class BaseFeed(Feed):
|
||||
feed_type = Atom1Feed
|
||||
@ -18,8 +17,8 @@ class BaseFeed(Feed):
|
||||
|
||||
def item_description(self, vlog):
|
||||
if vlog.overlay:
|
||||
txt = 'Version %s-%s [%s] of package %s ' % (vlog.version, vlog.revision,
|
||||
vlog.slot, vlog.package)
|
||||
txt = 'Version %s-%s [%s] of package %s ' % \
|
||||
(vlog.version, vlog.revision, vlog.slot, vlog.package)
|
||||
else:
|
||||
txt = 'Version %s of package %s ' % (vlog.version, vlog.package)
|
||||
if vlog.action == vlog.VERSION_REMOVED:
|
||||
@ -36,8 +35,10 @@ class BaseFeed(Feed):
|
||||
return txt
|
||||
|
||||
def item_link(self, vlog):
|
||||
kwargs = {'category' : vlog.package.category, 'package' : vlog.package.name }
|
||||
return reverse('djeuscan.views.package', kwargs=kwargs) + '#' + vlog.tag()
|
||||
kwargs = {'category': vlog.package.category,
|
||||
'package': vlog.package.name}
|
||||
return "%s#%s" % (reverse('djeuscan.views.package', kwargs=kwargs),
|
||||
vlog.tag())
|
||||
|
||||
def item_pubdate(self, vlog):
|
||||
return vlog.datetime
|
||||
@ -45,18 +46,20 @@ class BaseFeed(Feed):
|
||||
def item_categories(self, vlog):
|
||||
return [vlog.package.category]
|
||||
|
||||
|
||||
class GlobalFeed(BaseFeed):
|
||||
title = "euscan"
|
||||
link = "/"
|
||||
description = "Last euscan changes"
|
||||
|
||||
def categories(self):
|
||||
categories = Package.objects.values('category').distinct();
|
||||
categories = Package.objects.values('category').distinct()
|
||||
return [category['category'] for category in categories]
|
||||
|
||||
def items(self):
|
||||
return VersionLog.objects.order_by('-id')[:250]
|
||||
|
||||
|
||||
class PackageFeed(BaseFeed):
|
||||
feed_type = Atom1Feed
|
||||
|
||||
@ -67,7 +70,8 @@ class PackageFeed(BaseFeed):
|
||||
return "%s" % package
|
||||
|
||||
def link(self, package):
|
||||
return reverse('djeuscan.views.package', args=[package.category, package.name])
|
||||
return reverse('djeuscan.views.package', args=[package.category,
|
||||
package.name])
|
||||
|
||||
def description(self, package):
|
||||
return package.description
|
||||
@ -78,6 +82,7 @@ class PackageFeed(BaseFeed):
|
||||
def item_description(self, vlog):
|
||||
return ''
|
||||
|
||||
|
||||
class MaintainerFeed(BaseFeed):
|
||||
feed_type = Atom1Feed
|
||||
|
||||
@ -91,12 +96,14 @@ class MaintainerFeed(BaseFeed):
|
||||
return "Last changes for %s" % maintainer
|
||||
|
||||
def link(self, maintainer):
|
||||
return reverse('djeuscan.views.maintainer', kwargs={'maintainer_id' : maintainer.id})
|
||||
return reverse('djeuscan.views.maintainer',
|
||||
kwargs={'maintainer_id': maintainer.id})
|
||||
|
||||
def items(self, maintainer):
|
||||
q = VersionLog.objects.filter(package__maintainers__id=maintainer.id)
|
||||
return q.order_by('-id')[:50]
|
||||
|
||||
|
||||
class HerdFeed(BaseFeed):
|
||||
feed_type = Atom1Feed
|
||||
|
||||
@ -116,6 +123,7 @@ class HerdFeed(BaseFeed):
|
||||
q = VersionLog.objects.filter(package__herds__id=herd.id)
|
||||
return q.order_by('-id')[:100]
|
||||
|
||||
|
||||
class CategoryFeed(BaseFeed):
|
||||
feed_type = Atom1Feed
|
||||
|
||||
|
@ -1,7 +1,9 @@
|
||||
from django import forms
|
||||
|
||||
|
||||
class WorldForm(forms.Form):
|
||||
world = forms.FileField()
|
||||
|
||||
|
||||
class PackagesForm(forms.Form):
|
||||
packages = forms.CharField(widget=forms.Textarea)
|
||||
|
@ -1,8 +1,9 @@
|
||||
from optparse import make_option
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.core.management.base import BaseCommand
|
||||
from djeuscan.models import Package
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
help = 'List packages'
|
||||
|
@ -1,11 +1,8 @@
|
||||
import datetime
|
||||
|
||||
from optparse import make_option
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.core.management.base import BaseCommand
|
||||
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
|
||||
from djeuscan import charts
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
help = 'Regenerate rrd database'
|
||||
@ -15,10 +12,12 @@ class Command(BaseCommand):
|
||||
charts.rrd_update('world', wlog.datetime, wlog)
|
||||
|
||||
for clog in CategoryLog.objects.all():
|
||||
charts.rrd_update('category-%s' % clog.category, clog.datetime, clog)
|
||||
charts.rrd_update('category-%s' % clog.category,
|
||||
clog.datetime, clog)
|
||||
|
||||
for hlog in HerdLog.objects.all():
|
||||
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
|
||||
|
||||
for mlog in MaintainerLog.objects.all():
|
||||
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, mlog.datetime, mlog)
|
||||
charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
|
||||
mlog.datetime, mlog)
|
||||
|
@ -1,19 +1,15 @@
|
||||
import subprocess
|
||||
import portage
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
|
||||
from portage import versions
|
||||
from optparse import make_option
|
||||
|
||||
from django.db.transaction import commit_on_success
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.core.management.base import BaseCommand
|
||||
from djeuscan.models import Package, Herd, Maintainer
|
||||
|
||||
from gentoolkit.query import Query
|
||||
from gentoolkit.errors import GentoolkitFatalError
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
|
||||
@ -53,7 +49,9 @@ class Command(BaseCommand):
|
||||
)
|
||||
|
||||
if not matches:
|
||||
sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % query))
|
||||
sys.stderr.write(
|
||||
self.style.ERROR("Unknown package '%s'\n" % query)
|
||||
)
|
||||
return
|
||||
|
||||
matches = sorted(matches)
|
||||
@ -62,7 +60,9 @@ class Command(BaseCommand):
|
||||
pkg = matches.pop()
|
||||
|
||||
if not obj:
|
||||
obj, created = Package.objects.get_or_create(category=pkg.category, name=pkg.name)
|
||||
obj, created = Package.objects.get_or_create(
|
||||
category=pkg.category, name=pkg.name
|
||||
)
|
||||
else:
|
||||
created = False
|
||||
|
||||
@ -70,22 +70,34 @@ class Command(BaseCommand):
|
||||
obj.homepage = pkg.environment("HOMEPAGE")
|
||||
obj.description = pkg.environment("DESCRIPTION")
|
||||
except GentoolkitFatalError, err:
|
||||
sys.stderr.write(self.style.ERROR("Gentoolkit fatal error: '%s'\n" % str(err)))
|
||||
sys.stderr.write(
|
||||
self.style.ERROR(
|
||||
"Gentoolkit fatal error: '%s'\n" % str(err)
|
||||
)
|
||||
)
|
||||
|
||||
if created and not self.options['quiet']:
|
||||
sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
|
||||
|
||||
if pkg.metadata:
|
||||
herds = dict([(herd[0], herd) for herd in pkg.metadata.herds(True)])
|
||||
maintainers = dict([(m.email, m) for m in pkg.metadata.maintainers()])
|
||||
herds = dict(
|
||||
[(herd[0], herd) for herd in pkg.metadata.herds(True)]
|
||||
)
|
||||
maintainers = dict(
|
||||
[(m.email, m) for m in pkg.metadata.maintainers()]
|
||||
)
|
||||
|
||||
existing_herds = [h.herd for h in obj.herds.all()]
|
||||
new_herds = set(herds.keys()).difference(existing_herds)
|
||||
old_herds = set(existing_herds).difference(herds.keys())
|
||||
|
||||
existing_maintainers = [m.email for m in obj.maintainers.all()]
|
||||
new_maintainers = set(maintainers.keys()).difference(existing_maintainers)
|
||||
old_maintainers = set(existing_maintainers).difference(maintainers.keys())
|
||||
new_maintainers = set(
|
||||
maintainers.keys()).difference(existing_maintainers
|
||||
)
|
||||
old_maintainers = set(
|
||||
existing_maintainers).difference(maintainers.keys()
|
||||
)
|
||||
|
||||
for herd in obj.herds.all():
|
||||
if herd.herd in old_herds:
|
||||
@ -101,7 +113,9 @@ class Command(BaseCommand):
|
||||
|
||||
for maintainer in new_maintainers:
|
||||
maintainer = maintainers[maintainer]
|
||||
maintainer = self.store_maintainer(maintainer.name, maintainer.email)
|
||||
maintainer = self.store_maintainer(
|
||||
maintainer.name, maintainer.email
|
||||
)
|
||||
obj.maintainers.add(maintainer)
|
||||
|
||||
obj.save()
|
||||
@ -131,9 +145,12 @@ class Command(BaseCommand):
|
||||
|
||||
if created:
|
||||
if not self.options['quiet']:
|
||||
sys.stdout.write('+ [m] %s <%s>\n' % (name.encode('utf-8'), email))
|
||||
sys.stdout.write(
|
||||
'+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
|
||||
)
|
||||
|
||||
if not maintainer.name or name not in [maintainer.name, email, '{nil}']:
|
||||
if not maintainer.name or \
|
||||
name not in [maintainer.name, email, '{nil}']:
|
||||
maintainer.name = name
|
||||
maintainer.save()
|
||||
|
||||
|
@ -4,13 +4,13 @@ import sys
|
||||
import os
|
||||
import re
|
||||
|
||||
from portage import versions
|
||||
from optparse import make_option
|
||||
|
||||
from django.db.transaction import commit_on_success
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.core.management.base import BaseCommand
|
||||
from djeuscan.models import Package, Version, VersionLog
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
|
||||
@ -39,7 +39,8 @@ class Command(BaseCommand):
|
||||
action='store_true',
|
||||
dest='prefetch',
|
||||
default=False,
|
||||
help='Prefetch all versions and packages from DB to speedup full scan process.'),
|
||||
help=('Prefetch all versions and packages from DB to '
|
||||
'speedup full scan process.')),
|
||||
make_option('--quiet',
|
||||
action='store_true',
|
||||
dest='quiet',
|
||||
@ -59,22 +60,28 @@ class Command(BaseCommand):
|
||||
self._cache['packages'][key] = package
|
||||
|
||||
def cache_get_package(self, category, name):
|
||||
return self._cache['packages'].get(self.cache_hash_package(category, name))
|
||||
return self._cache['packages'].get(
|
||||
self.cache_hash_package(category, name)
|
||||
)
|
||||
|
||||
def cache_hash_version(self, category, name, version, revision, slot, overlay):
|
||||
def cache_hash_version(self, category, name, version, revision, slot,
|
||||
overlay):
|
||||
key = '%s/%s-%s-r%s %s %s' % (category, name,
|
||||
version, revision,
|
||||
slot, overlay)
|
||||
return key
|
||||
|
||||
def cache_get_version(self, category, name, version, revision, slot, overlay):
|
||||
key = self.cache_hash_version(category, name, version, revision, slot, overlay)
|
||||
def cache_get_version(self, category, name, version, revision, slot,
|
||||
overlay):
|
||||
key = self.cache_hash_version(category, name, version, revision, slot,
|
||||
overlay)
|
||||
return self._cache['versions'].get(key)
|
||||
|
||||
def cache_store_version(self, version):
|
||||
key = self.cache_hash_version(version.package.category, version.package.name,
|
||||
version.version, version.revision, version.slot,
|
||||
version.overlay)
|
||||
key = self.cache_hash_version(
|
||||
version.package.category, version.package.name, version.version,
|
||||
version.revision, version.slot, version.overlay
|
||||
)
|
||||
self._cache['versions'][key] = version
|
||||
|
||||
def handle(self, *args, **options):
|
||||
@ -119,7 +126,8 @@ class Command(BaseCommand):
|
||||
|
||||
cmd = ['eix', '-!']
|
||||
|
||||
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
|
||||
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
|
||||
communicate()[0]
|
||||
output = output.strip().strip('\n').split('\n')
|
||||
|
||||
overlay_re = re.compile(r'^\[(?P<key>\d+)] "(?P<name>.*?)"')
|
||||
@ -139,7 +147,8 @@ class Command(BaseCommand):
|
||||
env = os.environ
|
||||
env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
|
||||
|
||||
cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages', '-x']
|
||||
cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages',
|
||||
'-x']
|
||||
if query:
|
||||
cmd.extend(['--exact', query])
|
||||
|
||||
@ -151,7 +160,8 @@ class Command(BaseCommand):
|
||||
if not self.options['quiet']:
|
||||
self.stdout.write('done\n')
|
||||
|
||||
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
|
||||
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
|
||||
communicate()[0]
|
||||
output = output.strip().strip('\n')
|
||||
|
||||
if len(output) == 0:
|
||||
@ -166,13 +176,19 @@ class Command(BaseCommand):
|
||||
else:
|
||||
Package.objects.filter(name=query).delete()
|
||||
else:
|
||||
sys.stderr.write(self.style.ERROR("Unknown package '%s'\n" % query))
|
||||
sys.stderr.write(
|
||||
self.style.ERROR(
|
||||
"Unknown package '%s'\n" % query
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
output = output.split('\n')
|
||||
packages = {}
|
||||
|
||||
line_re = re.compile(r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$')
|
||||
line_re = re.compile(
|
||||
r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$'
|
||||
)
|
||||
|
||||
package = None
|
||||
|
||||
@ -190,7 +206,8 @@ class Command(BaseCommand):
|
||||
|
||||
packages['%s/%s' % (cat, pkg)] = True
|
||||
|
||||
if not package or not (cat == package.category and pkg == package.name):
|
||||
if not package or not \
|
||||
(cat == package.category and pkg == package.name):
|
||||
package = self.store_package(cat, pkg)
|
||||
|
||||
self.store_version(package, cpv, slot, overlay)
|
||||
@ -208,16 +225,23 @@ class Command(BaseCommand):
|
||||
obj = self.cache_get_package(cat, pkg)
|
||||
|
||||
if not obj:
|
||||
obj, created = Package.objects.get_or_create(category=cat, name=pkg)
|
||||
obj, created = Package.objects.get_or_create(
|
||||
category=cat,
|
||||
name=pkg
|
||||
)
|
||||
self.cache_store_package(obj)
|
||||
|
||||
if created:
|
||||
if not self.options['quiet']:
|
||||
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
|
||||
|
||||
' Set all versions dead, then set found versions alive and delete old versions '
|
||||
# Set all versions dead, then set found versions alive and
|
||||
# delete old versions
|
||||
if not self.options['all']:
|
||||
Version.objects.filter(package=obj, packaged=True).update(alive=False)
|
||||
Version.objects.filter(
|
||||
package=obj,
|
||||
packaged=True
|
||||
).update(alive=False)
|
||||
|
||||
return obj
|
||||
|
||||
@ -232,11 +256,15 @@ class Command(BaseCommand):
|
||||
overlay = 'gentoo'
|
||||
|
||||
created = False
|
||||
obj = self.cache_get_version(package.category, package.name, ver, rev, slot, overlay)
|
||||
obj = self.cache_get_version(
|
||||
package.category, package.name, ver, rev, slot, overlay
|
||||
)
|
||||
if not obj:
|
||||
obj, created = Version.objects.get_or_create(package=package, slot=slot,
|
||||
obj, created = Version.objects.get_or_create(
|
||||
package=package, slot=slot,
|
||||
revision=rev, version=ver,
|
||||
overlay=overlay)
|
||||
overlay=overlay
|
||||
)
|
||||
|
||||
obj.alive = True
|
||||
obj.packaged = True
|
||||
@ -245,7 +273,8 @@ class Command(BaseCommand):
|
||||
if created:
|
||||
self.cache_store_version(obj)
|
||||
|
||||
''' nothing to do (note: it can't be an upstream version because overlay can't be empty here) '''
|
||||
# nothing to do (note: it can't be an upstream version because
|
||||
# overlay can't be empty here)
|
||||
if not created:
|
||||
return
|
||||
|
||||
@ -287,8 +316,10 @@ class Command(BaseCommand):
|
||||
if self.options['no-log']:
|
||||
continue
|
||||
|
||||
entry = VersionLog.objects.create(package=version.package,
|
||||
action=VersionLog.VERSION_REMOVED)
|
||||
entry = VersionLog.objects.create(
|
||||
package=version.package,
|
||||
action=VersionLog.VERSION_REMOVED
|
||||
)
|
||||
entry.slot = version.slot
|
||||
entry.revision = version.revision
|
||||
entry.version = version.version
|
||||
@ -296,4 +327,3 @@ class Command(BaseCommand):
|
||||
entry.save()
|
||||
|
||||
Version.objects.filter(packaged=True, alive=False).delete()
|
||||
|
||||
|
@ -1,19 +1,17 @@
|
||||
import subprocess
|
||||
import portage
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
|
||||
from StringIO import StringIO
|
||||
from datetime import datetime
|
||||
from portage import versions
|
||||
from optparse import make_option
|
||||
|
||||
from django.utils import timezone
|
||||
from django.db.transaction import commit_on_success
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.core.management.base import BaseCommand
|
||||
from djeuscan.models import Package, Version, EuscanResult, VersionLog
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
|
||||
@ -83,8 +81,12 @@ class Command(BaseCommand):
|
||||
def parse_output(self, options, output):
|
||||
from portage.versions import _cp
|
||||
|
||||
package_re = re.compile(r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$')
|
||||
version_re = re.compile(r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$')
|
||||
package_re = re.compile(
|
||||
r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$'
|
||||
)
|
||||
version_re = re.compile(
|
||||
r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$'
|
||||
)
|
||||
|
||||
package = None
|
||||
log = ""
|
||||
@ -124,7 +126,6 @@ class Command(BaseCommand):
|
||||
obj.datetime = timezone.now()
|
||||
obj.save()
|
||||
|
||||
|
||||
def store_package(self, options, cpv):
|
||||
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
|
||||
|
||||
@ -133,15 +134,16 @@ class Command(BaseCommand):
|
||||
if created and not options['quiet']:
|
||||
sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
|
||||
|
||||
' Set all versions dead, then set found versions alive and delete old versions '
|
||||
# Set all versions dead, then set found versions alive and
|
||||
# delete old versions
|
||||
Version.objects.filter(package=obj, packaged=False).update(alive=False)
|
||||
|
||||
return obj
|
||||
|
||||
def store_version(self, options, package, ver, url):
|
||||
obj, created = Version.objects.get_or_create(package=package, slot='',
|
||||
revision='r0', version=ver,
|
||||
overlay='')
|
||||
obj, created = Version.objects.get_or_create(
|
||||
package=package, slot='', revision='r0', version=ver, overlay=''
|
||||
)
|
||||
|
||||
obj.alive = True
|
||||
obj.urls = url
|
||||
@ -155,7 +157,10 @@ class Command(BaseCommand):
|
||||
if not options['quiet']:
|
||||
sys.stdout.write('+ [u] %s %s\n' % (obj, url))
|
||||
|
||||
entry = VersionLog.objects.create(package=package, action=VersionLog.VERSION_ADDED)
|
||||
entry = VersionLog.objects.create(
|
||||
package=package,
|
||||
action=VersionLog.VERSION_ADDED
|
||||
)
|
||||
entry.slot = ''
|
||||
entry.revision = 'r0'
|
||||
entry.version = ver
|
||||
@ -165,12 +170,14 @@ class Command(BaseCommand):
|
||||
package.n_versions += 1
|
||||
package.save()
|
||||
|
||||
|
||||
@commit_on_success
|
||||
def purge_versions(self, options):
|
||||
' For each dead versions '
|
||||
for version in Version.objects.filter(packaged=False, alive=False):
|
||||
entry = VersionLog.objects.create(package=version.package, action=VersionLog.VERSION_REMOVED)
|
||||
entry = VersionLog.objects.create(
|
||||
package=version.package,
|
||||
action=VersionLog.VERSION_REMOVED
|
||||
)
|
||||
entry.slot = version.slot
|
||||
entry.revision = version.revision
|
||||
entry.version = version.version
|
||||
@ -183,4 +190,3 @@ class Command(BaseCommand):
|
||||
if not options['quiet']:
|
||||
sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
|
||||
Version.objects.filter(packaged=False, alive=False).delete()
|
||||
|
||||
|
@ -1,10 +1,7 @@
|
||||
import datetime
|
||||
|
||||
from optparse import make_option
|
||||
|
||||
from django.db.models import Count, Sum
|
||||
from django.db.transaction import commit_on_success
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils import timezone
|
||||
|
||||
from djeuscan.models import Package, Herd, Maintainer, Version
|
||||
@ -13,6 +10,7 @@ from djeuscan import charts
|
||||
|
||||
from distutils.version import StrictVersion, LooseVersion
|
||||
|
||||
|
||||
def compare_versions(version1, version2):
|
||||
try:
|
||||
return cmp(StrictVersion(version1), StrictVersion(version2))
|
||||
@ -20,6 +18,7 @@ def compare_versions(version1, version2):
|
||||
except ValueError:
|
||||
return cmp(LooseVersion(version1), LooseVersion(version2))
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
_overlays = {}
|
||||
help = 'Update counters'
|
||||
@ -98,7 +97,8 @@ class Command(BaseCommand):
|
||||
return
|
||||
if version['version'].startswith('9999'):
|
||||
return
|
||||
if compare_versions(storage[key]['version'], version['version']) < 0:
|
||||
if compare_versions(storage[key]['version'],
|
||||
version['version']) < 0:
|
||||
storage[key] = version
|
||||
|
||||
if not options['fast']:
|
||||
@ -126,15 +126,23 @@ class Command(BaseCommand):
|
||||
package.n_overlay = n_overlay.get(package.id, 0)
|
||||
|
||||
default = {'id': None}
|
||||
package.last_version_gentoo_id = last_versions_gentoo.get(package.id, default)['id']
|
||||
package.last_version_overlay_id = last_versions_overlay.get(package.id, default)['id']
|
||||
package.last_version_upstream_id = last_versions_upstream.get(package.id, default)['id']
|
||||
package.last_version_gentoo_id = last_versions_gentoo.get(
|
||||
package.id, default
|
||||
)['id']
|
||||
package.last_version_overlay_id = last_versions_overlay.get(
|
||||
package.id, default
|
||||
)['id']
|
||||
package.last_version_upstream_id = last_versions_upstream.get(
|
||||
package.id, default
|
||||
)['id']
|
||||
|
||||
package.save()
|
||||
|
||||
n_packages_gentoo = int(package.n_packaged == package.n_versions)
|
||||
n_packages_overlay = int(package.n_overlay and package.n_packaged + package.n_overlay == package.n_versions)
|
||||
n_packages_outdated = int(package.n_packaged + package.n_overlay < package.n_versions)
|
||||
n_packages_overlay = int(package.n_overlay and package.n_packaged \
|
||||
+ package.n_overlay == package.n_versions)
|
||||
n_packages_outdated = int(package.n_packaged + package.n_overlay \
|
||||
< package.n_versions)
|
||||
|
||||
def update_row(storage, key):
|
||||
storage[key].n_packages_gentoo += n_packages_gentoo
|
||||
@ -143,7 +151,10 @@ class Command(BaseCommand):
|
||||
|
||||
storage[key].n_versions_gentoo += package.n_packaged
|
||||
storage[key].n_versions_overlay += package.n_overlay
|
||||
storage[key].n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
|
||||
storage[key].n_versions_upstream += package.n_versions - \
|
||||
package.n_packaged - \
|
||||
package.n_overlay
|
||||
|
||||
def update_log(storage, qs):
|
||||
for row in qs:
|
||||
update_row(storage, row['id'])
|
||||
@ -159,7 +170,9 @@ class Command(BaseCommand):
|
||||
|
||||
wlog.n_versions_gentoo += package.n_packaged
|
||||
wlog.n_versions_overlay += package.n_overlay
|
||||
wlog.n_versions_upstream += package.n_versions - package.n_packaged - package.n_overlay
|
||||
wlog.n_versions_upstream += package.n_versions - \
|
||||
package.n_packaged - \
|
||||
package.n_overlay
|
||||
|
||||
if options['nolog']:
|
||||
return
|
||||
|
@ -1,5 +1,6 @@
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Herd(models.Model):
|
||||
herd = models.CharField(max_length=128, unique=True)
|
||||
email = models.CharField(max_length=128, blank=True, null=True)
|
||||
@ -9,6 +10,7 @@ class Herd(models.Model):
|
||||
return '%s <%s>' % (self.herd, self.email)
|
||||
return self.herd
|
||||
|
||||
|
||||
class Maintainer(models.Model):
|
||||
name = models.CharField(max_length=128)
|
||||
email = models.CharField(max_length=128, unique=True)
|
||||
@ -16,6 +18,7 @@ class Maintainer(models.Model):
|
||||
def __unicode__(self):
|
||||
return '%s <%s>' % (self.name, self.email)
|
||||
|
||||
|
||||
class Package(models.Model):
|
||||
category = models.CharField(max_length=128)
|
||||
name = models.CharField(max_length=128)
|
||||
@ -30,15 +33,18 @@ class Package(models.Model):
|
||||
n_overlay = models.IntegerField(default=0)
|
||||
|
||||
' And we also pre-compute last versions '
|
||||
last_version_gentoo = models.ForeignKey('Version', blank=True, null=True,
|
||||
related_name="last_version_gentoo",
|
||||
on_delete=models.SET_NULL)
|
||||
last_version_overlay = models.ForeignKey('Version', blank=True, null=True,
|
||||
related_name="last_version_overlay",
|
||||
on_delete=models.SET_NULL)
|
||||
last_version_upstream = models.ForeignKey('Version', blank=True, null=True,
|
||||
related_name="last_version_upstream",
|
||||
on_delete=models.SET_NULL)
|
||||
last_version_gentoo = models.ForeignKey(
|
||||
'Version', blank=True, null=True, related_name="last_version_gentoo",
|
||||
on_delete=models.SET_NULL
|
||||
)
|
||||
last_version_overlay = models.ForeignKey(
|
||||
'Version', blank=True, null=True, related_name="last_version_overlay",
|
||||
on_delete=models.SET_NULL
|
||||
)
|
||||
last_version_upstream = models.ForeignKey(
|
||||
'Version', blank=True, null=True, related_name="last_version_upstream",
|
||||
on_delete=models.SET_NULL
|
||||
)
|
||||
|
||||
def __unicode__(self):
|
||||
return '%s/%s' % (self.category, self.name)
|
||||
@ -46,6 +52,7 @@ class Package(models.Model):
|
||||
class Meta:
|
||||
unique_together = ['category', 'name']
|
||||
|
||||
|
||||
class Version(models.Model):
|
||||
package = models.ForeignKey(Package)
|
||||
slot = models.CharField(max_length=128)
|
||||
@ -57,13 +64,15 @@ class Version(models.Model):
|
||||
alive = models.BooleanField(default=True, db_index=True)
|
||||
|
||||
def __unicode__(self):
|
||||
return '%s/%s-%s-%s:%s [%s]' % (self.package.category, self.package.name,
|
||||
self.version, self.revision, self.slot,
|
||||
self.overlay)
|
||||
return '%s/%s-%s-%s:%s [%s]' % (
|
||||
self.package.category, self.package.name, self.version,
|
||||
self.revision, self.slot, self.overlay
|
||||
)
|
||||
|
||||
class Meta:
|
||||
unique_together = ['package', 'slot', 'revision', 'version', 'overlay']
|
||||
|
||||
|
||||
class VersionLog(models.Model):
|
||||
VERSION_ADDED = 1
|
||||
VERSION_REMOVED = 2
|
||||
@ -87,16 +96,20 @@ class VersionLog(models.Model):
|
||||
|
||||
def __unicode__(self):
|
||||
txt = '+ ' if self.action == self.VERSION_ADDED else '- '
|
||||
txt += '%s/%s-%s-%s:%s [%s]' % (self.package.category, self.package.name,
|
||||
self.version, self.revision, self.slot,
|
||||
self.overlay if self.overlay else '<upstream>')
|
||||
txt += '%s/%s-%s-%s:%s [%s]' % (
|
||||
self.package.category, self.package.name, self.version,
|
||||
self.revision, self.slot,
|
||||
self.overlay if self.overlay else '<upstream>'
|
||||
)
|
||||
return txt
|
||||
|
||||
|
||||
class EuscanResult(models.Model):
|
||||
package = models.ForeignKey(Package)
|
||||
datetime = models.DateTimeField()
|
||||
result = models.TextField(blank=True)
|
||||
|
||||
|
||||
# Keep data for charts
|
||||
class Log(models.Model):
|
||||
datetime = models.DateTimeField()
|
||||
@ -116,29 +129,34 @@ class Log(models.Model):
|
||||
n_versions_upstream = models.IntegerField(default=0)
|
||||
|
||||
def __unicode__(self):
|
||||
return u'[%d:%d:%d] [%d:%d:%d]' % \
|
||||
(self.n_packages_gentoo, self.n_packages_overlay, self.n_packages_outdated, \
|
||||
self.n_versions_gentoo, self.n_versions_overlay, self.n_versions_upstream)
|
||||
return u'[%d:%d:%d] [%d:%d:%d]' % (
|
||||
self.n_packages_gentoo, self.n_packages_overlay,
|
||||
self.n_packages_outdated, self.n_versions_gentoo,
|
||||
self.n_versions_overlay, self.n_versions_upstream
|
||||
)
|
||||
|
||||
|
||||
class WorldLog(Log):
|
||||
def __unicode__(self):
|
||||
return u'world ' + Log.__unicode__(self)
|
||||
|
||||
|
||||
class CategoryLog(Log):
|
||||
category = models.CharField(max_length=128)
|
||||
|
||||
def __unicode__(self):
|
||||
return u'%s %s' % (self.category, Log.__unicode__(self))
|
||||
|
||||
|
||||
class HerdLog(Log):
|
||||
herd = models.ForeignKey(Herd)
|
||||
|
||||
def __unicode__(self):
|
||||
return u'%s %s' % (self.herd, Log.__unicode__(self))
|
||||
|
||||
|
||||
class MaintainerLog(Log):
|
||||
maintainer = models.ForeignKey(Maintainer)
|
||||
|
||||
def __unicode__(self):
|
||||
return u'%s %s' % (self.maintainer, Log.__unicode__(self))
|
||||
|
||||
|
@ -2,6 +2,7 @@ from django import template
|
||||
|
||||
register = template.Library()
|
||||
|
||||
|
||||
def div(value, arg=None):
|
||||
return float(value) / float(arg)
|
||||
|
||||
|
@ -1,11 +1,14 @@
|
||||
from django.template import Node, Library
|
||||
from django.template import Library
|
||||
import math
|
||||
|
||||
register = Library()
|
||||
|
||||
# taken from http://lybniz2.sourceforge.net/safeeval.html
|
||||
# make a list of safe functions
|
||||
math_safe_list = ['acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'cosh', 'degrees', 'e', 'exp', 'fabs', 'floor', 'fmod', 'frexp', 'hypot', 'ldexp', 'log', 'log10', 'modf', 'pi', 'pow', 'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh']
|
||||
math_safe_list = ['acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'cosh',
|
||||
'degrees', 'e', 'exp', 'fabs', 'floor', 'fmod', 'frexp',
|
||||
'hypot', 'ldexp', 'log', 'log10', 'modf', 'pi', 'pow',
|
||||
'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh']
|
||||
|
||||
# use the list to filter the local namespace
|
||||
math_safe_dict = dict([(k, getattr(math, k)) for k in math_safe_list])
|
||||
@ -13,11 +16,13 @@ math_safe_dict = dict([(k, getattr(math, k)) for k in math_safe_list])
|
||||
# add any needed builtins back in.
|
||||
math_safe_dict['abs'] = abs
|
||||
|
||||
|
||||
@register.filter('math')
|
||||
def math_(lopr, expr):
|
||||
"""Evals a math expression and returns it's value.
|
||||
|
||||
"$1" is a placeholder. Insert "$1" in the expression where the value is to be used. All math functions such as abs, sin, cos, floor are supported.
|
||||
"$1" is a placeholder. Insert "$1" in the expression where the value is
|
||||
to be used. All math functions such as abs, sin, cos, floor are supported.
|
||||
Example,
|
||||
a. You will be redirected in {{ seconds|math:"$1 / 60.0" }} minutes
|
||||
b. Square of {{ x }} is {{ x|math:"$1 * $1" }}
|
||||
@ -25,5 +30,6 @@ def math_(lopr, expr):
|
||||
d. Given x = {{ x }}, (2 + x) * 6 = {{ x|math:"(2 + $1) * 6" }}
|
||||
"""
|
||||
if lopr:
|
||||
return eval(expr.replace('$1', str(lopr)), {"__builtins__": None}, math_safe_dict)
|
||||
return eval(expr.replace('$1', str(lopr)), {"__builtins__": None},
|
||||
math_safe_dict)
|
||||
return ''
|
||||
|
@ -2,6 +2,7 @@ from django import template
|
||||
|
||||
register = template.Library()
|
||||
|
||||
|
||||
def mul(value, arg=None):
|
||||
return value * arg
|
||||
|
||||
|
@ -2,16 +2,19 @@ from django import template
|
||||
|
||||
register = template.Library()
|
||||
|
||||
|
||||
@register.inclusion_tag('euscan/_packages.html', takes_context=True)
|
||||
def packages(context, packages):
|
||||
context['packages'] = packages
|
||||
return context
|
||||
|
||||
|
||||
@register.inclusion_tag('euscan/_package_cols.html', takes_context=True)
|
||||
def package_cols(context, infos):
|
||||
context['infos'] = infos
|
||||
return context
|
||||
|
||||
|
||||
@register.inclusion_tag('euscan/_package_bar.html', takes_context=True)
|
||||
def package_bar(context, infos):
|
||||
context['infos'] = infos
|
||||
|
@ -2,6 +2,7 @@ from django import template
|
||||
|
||||
register = template.Library()
|
||||
|
||||
|
||||
def sub(value, arg=None):
|
||||
return value - arg
|
||||
|
||||
|
@ -6,6 +6,7 @@ from datetime import datetime
|
||||
|
||||
register = template.Library()
|
||||
|
||||
|
||||
def timedelta(value, arg=None):
|
||||
if not value:
|
||||
return ''
|
||||
|
@ -7,6 +7,7 @@ Replace these with more appropriate tests for your application.
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
|
||||
class SimpleTest(TestCase):
|
||||
def test_basic_addition(self):
|
||||
"""
|
||||
@ -20,4 +21,3 @@ Another way to test that 1 + 1 is equal to 2.
|
||||
>>> 1 + 1 == 2
|
||||
True
|
||||
"""}
|
||||
|
||||
|
@ -1,29 +1,36 @@
|
||||
from django.conf.urls.defaults import *
|
||||
from feeds import *
|
||||
from django.conf.urls.defaults import url, patterns, include
|
||||
from feeds import PackageFeed, CategoryFeed, HerdFeed, MaintainerFeed, \
|
||||
GlobalFeed
|
||||
|
||||
package_patterns = patterns('djeuscan.views',
|
||||
url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/feed/$', PackageFeed(), name='package_feed'),
|
||||
url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/feed/$',
|
||||
PackageFeed(), name='package_feed'),
|
||||
(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/$', 'package'),
|
||||
)
|
||||
|
||||
categories_patterns = patterns('djeuscan.views',
|
||||
(r'^(?P<category>[\w+][\w+.-]*)/(view/)?$', 'category'),
|
||||
url(r'^(?P<category>[\w+][\w+.-]*)/feed/$', CategoryFeed(), name='category_feed'),
|
||||
(r'^(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', 'chart_category'),
|
||||
url(r'^(?P<category>[\w+][\w+.-]*)/feed/$', CategoryFeed(),
|
||||
name='category_feed'),
|
||||
(r'^(?P<category>[\w+][\w+.-]*)/charts/(?P<chart>[\w\-]+).png$',
|
||||
'chart_category'),
|
||||
(r'^$', 'categories'),
|
||||
)
|
||||
|
||||
herds_patterns = patterns('djeuscan.views',
|
||||
(r'^(?P<herd>[\@\{\}\w+.-]*)/(view/)?$', 'herd'),
|
||||
url(r'^(?P<herd>[\@\{\}\w+.-]*)/feed/$', HerdFeed(), name='herd_feed'),
|
||||
(r'^(?P<herd>[\@\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$', 'chart_herd'),
|
||||
(r'^(?P<herd>[\@\{\}\w+.-]*)/charts/(?P<chart>[\w\-]+).png$',
|
||||
'chart_herd'),
|
||||
(r'^$', 'herds'),
|
||||
)
|
||||
|
||||
maintainers_patterns = patterns('djeuscan.views',
|
||||
(r'^(?P<maintainer_id>\d+)/(view/)?$', 'maintainer'),
|
||||
url(r'^(?P<maintainer_id>\d+)/feed/$', MaintainerFeed(), name='maintainer_feed'),
|
||||
(r'^(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$', 'chart_maintainer'),
|
||||
url(r'^(?P<maintainer_id>\d+)/feed/$', MaintainerFeed(),
|
||||
name='maintainer_feed'),
|
||||
(r'^(?P<maintainer_id>\d+)/charts/(?P<chart>[\w\-]+).png$',
|
||||
'chart_maintainer'),
|
||||
(r'^$', 'maintainers'),
|
||||
)
|
||||
|
||||
|
@ -1,5 +1,5 @@
|
||||
from annoying.decorators import render_to
|
||||
from django.http import HttpResponse, Http404
|
||||
from django.http import Http404
|
||||
from django.shortcuts import get_object_or_404
|
||||
from django.db.models import Sum, Max
|
||||
|
||||
@ -10,78 +10,110 @@ import charts
|
||||
|
||||
""" Views """
|
||||
|
||||
|
||||
@render_to('euscan/index.html')
|
||||
def index(request):
|
||||
ctx = {}
|
||||
ctx['n_packaged'] = charts.xint(Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum'])
|
||||
ctx['n_overlay'] = charts.xint(Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum'])
|
||||
ctx['n_versions'] = charts.xint(Package.objects.aggregate(Sum('n_versions'))['n_versions__sum'])
|
||||
ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - ctx['n_overlay']
|
||||
ctx['n_packaged'] = charts.xint(
|
||||
Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum']
|
||||
)
|
||||
ctx['n_overlay'] = charts.xint(
|
||||
Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum']
|
||||
)
|
||||
ctx['n_versions'] = charts.xint(
|
||||
Package.objects.aggregate(Sum('n_versions'))['n_versions__sum']
|
||||
)
|
||||
ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - \
|
||||
ctx['n_overlay']
|
||||
ctx['n_packages'] = Package.objects.count()
|
||||
ctx['n_herds'] = Herd.objects.count()
|
||||
ctx['n_maintainers'] = Maintainer.objects.count()
|
||||
|
||||
try:
|
||||
ctx['last_scan'] = EuscanResult.objects.get(id=EuscanResult.objects.aggregate(Max('id'))['id__max']).datetime
|
||||
ctx['last_scan'] = EuscanResult.objects.get(
|
||||
id=EuscanResult.objects.aggregate(Max('id'))['id__max']
|
||||
).datetime
|
||||
except EuscanResult.DoesNotExist:
|
||||
ctx['last_scan'] = None
|
||||
|
||||
return ctx
|
||||
|
||||
|
||||
@render_to('euscan/logs.html')
|
||||
def logs(request):
|
||||
return {}
|
||||
|
||||
|
||||
@render_to('euscan/categories.html')
|
||||
def categories(request):
|
||||
categories = Package.objects.values('category').annotate(n_packaged=Sum('n_packaged'),
|
||||
categories = Package.objects.values('category').annotate(
|
||||
n_packaged=Sum('n_packaged'),
|
||||
n_overlay=Sum('n_overlay'),
|
||||
n_versions=Sum('n_versions'))
|
||||
n_versions=Sum('n_versions')
|
||||
)
|
||||
|
||||
return {'categories': categories}
|
||||
|
||||
|
||||
@render_to('euscan/category.html')
|
||||
def category(request, category):
|
||||
packages = Package.objects.filter(category=category)
|
||||
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
|
||||
packages = packages.select_related(
|
||||
'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
|
||||
)
|
||||
print dir(packages[0])
|
||||
if not packages:
|
||||
raise Http404
|
||||
return {'category': category, 'packages': packages}
|
||||
|
||||
|
||||
@render_to('euscan/herds.html')
|
||||
def herds(request):
|
||||
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of 'INNER JOIN'
|
||||
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of
|
||||
# 'INNER JOIN'
|
||||
herds = Package.objects.filter(herds__isnull=False)
|
||||
herds = herds.values('herds__herd').annotate(n_packaged=Sum('n_packaged'),
|
||||
herds = herds.values('herds__herd').annotate(
|
||||
n_packaged=Sum('n_packaged'),
|
||||
n_overlay=Sum('n_overlay'),
|
||||
n_versions=Sum('n_versions'))
|
||||
return {'herds': herds}
|
||||
|
||||
|
||||
@render_to('euscan/herd.html')
|
||||
def herd(request, herd):
|
||||
herd = get_object_or_404(Herd, herd=herd)
|
||||
packages = Package.objects.filter(herds__id=herd.id)
|
||||
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
|
||||
packages = packages.select_related(
|
||||
'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
|
||||
)
|
||||
return {'herd': herd, 'packages': packages}
|
||||
|
||||
|
||||
@render_to('euscan/maintainers.html')
|
||||
def maintainers(request):
|
||||
maintainers = Package.objects.filter(maintainers__isnull=False)
|
||||
maintainers = maintainers.values('maintainers__id', 'maintainers__name', 'maintainers__email')
|
||||
maintainers = maintainers.annotate(n_packaged=Sum('n_packaged'),
|
||||
maintainers = maintainers.values(
|
||||
'maintainers__id', 'maintainers__name', 'maintainers__email'
|
||||
)
|
||||
maintainers = maintainers.annotate(
|
||||
n_packaged=Sum('n_packaged'),
|
||||
n_overlay=Sum('n_overlay'),
|
||||
n_versions=Sum('n_versions'))
|
||||
n_versions=Sum('n_versions')
|
||||
)
|
||||
|
||||
return {'maintainers': maintainers}
|
||||
|
||||
|
||||
@render_to('euscan/maintainer.html')
|
||||
def maintainer(request, maintainer_id):
|
||||
maintainer = get_object_or_404(Maintainer, id=maintainer_id)
|
||||
packages = Package.objects.filter(maintainers__id=maintainer.id)
|
||||
packages = packages.select_related('last_version_gentoo', 'last_version_overlay', 'last_version_upstream')
|
||||
packages = packages.select_related(
|
||||
'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
|
||||
)
|
||||
return {'maintainer': maintainer, 'packages': packages}
|
||||
|
||||
|
||||
@render_to('euscan/overlays.html')
|
||||
def overlays(request):
|
||||
overlays = Package.objects.values('version__overlay')
|
||||
@ -89,6 +121,7 @@ def overlays(request):
|
||||
overlays = overlays.distinct()
|
||||
return {'overlays': overlays}
|
||||
|
||||
|
||||
@render_to('euscan/overlay.html')
|
||||
def overlay(request, overlay):
|
||||
packages = Package.objects.values('id', 'name', 'category',
|
||||
@ -99,6 +132,7 @@ def overlay(request, overlay):
|
||||
raise Http404
|
||||
return {'overlay': overlay, 'packages': packages}
|
||||
|
||||
|
||||
@render_to('euscan/package.html')
|
||||
def package(request, category, package):
|
||||
|
||||
@ -120,12 +154,15 @@ def package(request, category, package):
|
||||
packaged = sorted(packaged, key=version_key)
|
||||
upstream = sorted(upstream, key=version_key)
|
||||
|
||||
log = EuscanResult.objects.filter(package=package).order_by('-datetime')[:1]
|
||||
log = EuscanResult.objects.filter(package=package).\
|
||||
order_by('-datetime')[:1]
|
||||
log = log[0] if log else None
|
||||
vlog = VersionLog.objects.filter(package=package).order_by('-id')
|
||||
|
||||
return {'package': package, 'packaged': packaged,
|
||||
'upstream': upstream, 'log': log, 'vlog': vlog}
|
||||
|
||||
|
||||
@render_to('euscan/world.html')
|
||||
def world(request):
|
||||
world_form = WorldForm()
|
||||
@ -134,6 +171,7 @@ def world(request):
|
||||
return {'world_form': world_form,
|
||||
'packages_form': packages_form}
|
||||
|
||||
|
||||
@render_to('euscan/world_scan.html')
|
||||
def world_scan(request):
|
||||
packages = []
|
||||
@ -164,21 +202,27 @@ def world_scan(request):
|
||||
def about(request):
|
||||
return {}
|
||||
|
||||
|
||||
@render_to("euscan/api.html")
|
||||
def api(request):
|
||||
return {}
|
||||
|
||||
|
||||
@render_to("euscan/statistics.html")
|
||||
def statistics(request):
|
||||
return {}
|
||||
|
||||
|
||||
def chart(request, **kwargs):
|
||||
from django.views.static import serve
|
||||
|
||||
chart = kwargs['chart'] if 'chart' in kwargs else None
|
||||
|
||||
if 'maintainer_id' in kwargs:
|
||||
kwargs['maintainer'] = get_object_or_404(Maintainer, id=kwargs['maintainer_id'])
|
||||
kwargs['maintainer'] = get_object_or_404(
|
||||
Maintainer,
|
||||
id=kwargs['maintainer_id']
|
||||
)
|
||||
if 'herd' in kwargs:
|
||||
kwargs['herd'] = get_object_or_404(Herd, herd=kwargs['herd'])
|
||||
|
||||
@ -202,11 +246,14 @@ def chart(request, **kwargs):
|
||||
|
||||
return serve(request, path, document_root=charts.CHARTS_ROOT)
|
||||
|
||||
|
||||
def chart_maintainer(request, **kwargs):
|
||||
return chart(request, **kwargs)
|
||||
|
||||
|
||||
def chart_herd(request, **kwargs):
|
||||
return chart(request, **kwargs)
|
||||
|
||||
|
||||
def chart_category(request, **kwargs):
|
||||
return chart(request, **kwargs)
|
||||
|
@ -22,4 +22,3 @@ if settings.DEBUG:
|
||||
'document_root': os.path.join(settings.EUSCAN_ROOT, 'htdocs'),
|
||||
}),
|
||||
)
|
||||
|
||||
|
@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python
|
||||
import os, sys
|
||||
import os
|
||||
import sys
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "euscanwww.settings")
|
||||
|
@ -1,4 +1,5 @@
|
||||
import os, sys
|
||||
import os
|
||||
import sys
|
||||
sys.path.insert(0, '../')
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'euscanwww.settings'
|
||||
|
||||
|
@ -103,12 +103,14 @@
|
||||
{% endif %}
|
||||
{% if version.overlay == "gentoo" %}
|
||||
<img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
|
||||
{% else %}{% if version.overlay %}
|
||||
{% elif version.overlay %}
|
||||
<img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
|
||||
{% else %}
|
||||
<img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
|
||||
{% endif %}{% endif %}
|
||||
{% endif %}
|
||||
|
||||
{{ version }} - {{ version.datetime }}
|
||||
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
|
@ -5,7 +5,6 @@
|
||||
|
||||
__version__ = "git"
|
||||
|
||||
import sys
|
||||
|
||||
from portage.output import EOutput
|
||||
|
||||
@ -27,7 +26,8 @@ CONFIG = {
|
||||
output = EOutput(CONFIG['quiet'])
|
||||
|
||||
BLACKLIST_VERSIONS = [
|
||||
# Compatibility package for running binaries linked against a pre gcc 3.4 libstdc++, won't be updated
|
||||
# Compatibility package for running binaries linked against a
|
||||
# pre gcc 3.4 libstdc++, won't be updated
|
||||
'>=sys-libs/libstdc++-v3-3.4',
|
||||
]
|
||||
|
||||
@ -44,7 +44,10 @@ SCANDIR_BLACKLIST_URLS = [
|
||||
]
|
||||
|
||||
BRUTEFORCE_BLACKLIST_PACKAGES = [
|
||||
'net-zope/plonepopoll' # infinite loop any http://plone.org/products/plonepopoll/releases/*/plonepopoll-2-6-1.tgz link will work
|
||||
# infinite loop any
|
||||
# http://plone.org/products/plonepopoll/releases/*/plonepopoll-2-6-1.tgz
|
||||
# link will work
|
||||
'net-zope/plonepopoll'
|
||||
]
|
||||
|
||||
BRUTEFORCE_BLACKLIST_URLS = [
|
||||
|
@ -2,18 +2,21 @@ from euscan.handlers import generic, php, pypi, rubygem, kde, cpan
|
||||
|
||||
handlers = [kde, php, pypi, rubygem, cpan, generic]
|
||||
|
||||
|
||||
def find_best_handler(cpv, url):
|
||||
for handler in handlers:
|
||||
if handler.can_handle(cpv, url):
|
||||
return handler
|
||||
return None
|
||||
|
||||
|
||||
def scan(cpv, url):
|
||||
handler = find_best_handler(cpv, url)
|
||||
if handler:
|
||||
return handler.scan(cpv, url)
|
||||
return []
|
||||
|
||||
|
||||
def brute_force(cpv, url):
|
||||
handler = find_best_handler(cpv, url)
|
||||
if handler:
|
||||
|
@ -8,9 +8,11 @@ import euscan
|
||||
|
||||
_cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*")
|
||||
|
||||
|
||||
def can_handle(cpv, url):
|
||||
return url.startswith('mirror://cpan/')
|
||||
|
||||
|
||||
def guess_package(cp, url):
|
||||
match = _cpan_package_name_re.search(url)
|
||||
|
||||
@ -27,6 +29,7 @@ def guess_package(cp, url):
|
||||
|
||||
return pkg
|
||||
|
||||
|
||||
def gentoo_mangle_version(up_pv):
|
||||
pv = ""
|
||||
|
||||
@ -39,17 +42,20 @@ def gentoo_mangle_version(up_pv):
|
||||
c = up_pv[i]
|
||||
pv += c
|
||||
digits += int(c.isdigit())
|
||||
if c == '.': digits = 0
|
||||
if c == '.':
|
||||
digits = 0
|
||||
else:
|
||||
pv = up_pv
|
||||
|
||||
return helpers.gentoo_mangle_version(pv)
|
||||
|
||||
|
||||
def cpan_trim_version(pv):
|
||||
pv = re.sub('^[a-zA-Z]+', '', pv)
|
||||
pv = re.sub('[a-zA-Z]$', '', pv)
|
||||
return pv
|
||||
|
||||
|
||||
def cpan_mangle_version(pv):
|
||||
pos = pv.find('.')
|
||||
if pos < 0:
|
||||
@ -59,6 +65,7 @@ def cpan_mangle_version(pv):
|
||||
up_pv = cpan_trim_version(up_pv)
|
||||
return up_pv
|
||||
|
||||
|
||||
def cpan_vercmp(cp, a, b):
|
||||
try:
|
||||
return float(a) - float(b)
|
||||
@ -68,6 +75,7 @@ def cpan_vercmp(cp, a, b):
|
||||
else:
|
||||
return 1
|
||||
|
||||
|
||||
def scan(cpv, url):
|
||||
cp, ver, rev = portage.pkgsplit(cpv)
|
||||
pkg = guess_package(cp, url)
|
||||
@ -107,8 +115,12 @@ def scan(cpv, url):
|
||||
if helpers.version_filtered(cp, up_ver, up_pv, cpan_vercmp):
|
||||
continue
|
||||
|
||||
url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % \
|
||||
(version['cpanid'][0], version['cpanid'][0:1], version['cpanid'], version['archive'])
|
||||
url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % (
|
||||
version['cpanid'][0],
|
||||
version['cpanid'][0:1],
|
||||
version['cpanid'],
|
||||
version['archive']
|
||||
)
|
||||
|
||||
if url == orig_url:
|
||||
continue
|
||||
@ -117,5 +129,6 @@ def scan(cpv, url):
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def brute_force(cpv, url):
|
||||
return []
|
||||
|
@ -6,10 +6,12 @@ from BeautifulSoup import BeautifulSoup
|
||||
|
||||
import portage
|
||||
|
||||
from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS
|
||||
from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \
|
||||
BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS
|
||||
from euscan import helpers
|
||||
import euscan
|
||||
|
||||
|
||||
def scan_html(data, url, pattern):
|
||||
soup = BeautifulSoup(data)
|
||||
results = []
|
||||
@ -28,6 +30,7 @@ def scan_html(data, url, pattern):
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def scan_ftp(data, url, pattern):
|
||||
buf = StringIO.StringIO(data)
|
||||
results = []
|
||||
@ -40,6 +43,7 @@ def scan_ftp(data, url, pattern):
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
|
||||
if not steps:
|
||||
return []
|
||||
@ -91,6 +95,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
|
||||
|
||||
return versions
|
||||
|
||||
|
||||
def scan(cpv, url):
|
||||
for bu in SCANDIR_BLACKLIST_URLS:
|
||||
if re.match(bu, url):
|
||||
@ -107,13 +112,17 @@ def scan(cpv, url):
|
||||
if ver not in resolved_url:
|
||||
newver = helpers.version_change_end_sep(ver)
|
||||
if newver and newver in resolved_url:
|
||||
euscan.output.einfo("Version: using %s instead of %s" % (newver, ver))
|
||||
euscan.output.einfo(
|
||||
"Version: using %s instead of %s" % (newver, ver)
|
||||
)
|
||||
ver = newver
|
||||
|
||||
template = helpers.template_from_url(resolved_url, ver)
|
||||
if '${' not in template:
|
||||
euscan.output.einfo("Url doesn't seems to depend on version: %s not found in %s"
|
||||
% (ver, resolved_url))
|
||||
euscan.output.einfo(
|
||||
"Url doesn't seems to depend on version: %s not found in %s" %
|
||||
(ver, resolved_url)
|
||||
)
|
||||
return []
|
||||
else:
|
||||
euscan.output.einfo("Scanning: %s" % template)
|
||||
@ -121,6 +130,7 @@ def scan(cpv, url):
|
||||
steps = helpers.generate_scan_paths(template)
|
||||
return scan_directory_recursive(cp, ver, rev, "", steps, url)
|
||||
|
||||
|
||||
def brute_force(cpv, url):
|
||||
cp, ver, rev = portage.pkgsplit(cpv)
|
||||
|
||||
@ -155,8 +165,9 @@ def brute_force(cpv, url):
|
||||
template = helpers.template_from_url(url, ver)
|
||||
|
||||
if '${PV}' not in template:
|
||||
euscan.output.einfo("Url doesn't seems to depend on full version: %s not found in %s"
|
||||
% (ver, url))
|
||||
euscan.output.einfo(
|
||||
"Url doesn't seems to depend on full version: %s not found in %s" %
|
||||
(ver, url))
|
||||
return []
|
||||
else:
|
||||
euscan.output.einfo("Brute forcing: %s" % template)
|
||||
@ -187,11 +198,14 @@ def brute_force(cpv, url):
|
||||
result.append([url, version])
|
||||
|
||||
if len(result) > CONFIG['brute-force-false-watermark']:
|
||||
euscan.output.einfo("Broken server detected ! Skipping brute force.")
|
||||
euscan.output.einfo(
|
||||
"Broken server detected ! Skipping brute force."
|
||||
)
|
||||
return []
|
||||
|
||||
if CONFIG["brute-force-recursive"]:
|
||||
for v in helpers.gen_versions(list(components), CONFIG["brute-force"]):
|
||||
for v in helpers.gen_versions(list(components),
|
||||
CONFIG["brute-force"]):
|
||||
if v not in versions and tuple(v) not in done:
|
||||
versions.append(v)
|
||||
|
||||
@ -200,5 +214,6 @@ def brute_force(cpv, url):
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def can_handle(cpv, url):
|
||||
return True
|
||||
|
@ -1,10 +1,12 @@
|
||||
from euscan.handlers import generic
|
||||
|
||||
|
||||
def can_handle(cpv, url):
|
||||
if url.startswith('mirror://kde/'):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def clean_results(results):
|
||||
ret = []
|
||||
|
||||
@ -15,6 +17,7 @@ def clean_results(results):
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def scan(cpv, url):
|
||||
results = generic.scan(cpv, url)
|
||||
|
||||
@ -24,6 +27,7 @@ def scan(cpv, url):
|
||||
|
||||
return clean_results(results)
|
||||
|
||||
|
||||
def brute_force(cpv, url):
|
||||
results = generic.brute_force(cpv, url)
|
||||
|
||||
|
@ -6,6 +6,7 @@ import xml.dom.minidom
|
||||
from euscan import helpers
|
||||
import euscan
|
||||
|
||||
|
||||
def can_handle(cpv, url):
|
||||
if url.startswith('http://pear.php.net/get/'):
|
||||
return True
|
||||
@ -13,6 +14,7 @@ def can_handle(cpv, url):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def guess_package_and_channel(cp, url):
|
||||
match = re.search('http://(.*)/get/(.*)-(.*).tgz', url)
|
||||
|
||||
@ -24,6 +26,7 @@ def guess_package_and_channel(cp, url):
|
||||
|
||||
return pkg, host
|
||||
|
||||
|
||||
def scan(cpv, url):
|
||||
cp, ver, rev = portage.pkgsplit(cpv)
|
||||
pkg, channel = guess_package_and_channel(cp, url)
|
||||
@ -65,5 +68,6 @@ def scan(cpv, url):
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def brute_force(cpv, url):
|
||||
return []
|
||||
|
@ -1,5 +1,4 @@
|
||||
import xmlrpclib
|
||||
import pprint
|
||||
import re
|
||||
|
||||
import portage
|
||||
@ -7,9 +6,11 @@ import portage
|
||||
from euscan import helpers
|
||||
import euscan
|
||||
|
||||
|
||||
def can_handle(cpv, url):
|
||||
return url.startswith('mirror://pypi/')
|
||||
|
||||
|
||||
def guess_package(cp, url):
|
||||
match = re.search('mirror://pypi/\w+/(.*)/.*', url)
|
||||
if match:
|
||||
@ -19,10 +20,10 @@ def guess_package(cp, url):
|
||||
|
||||
return pkg
|
||||
|
||||
|
||||
def scan(cpv, url):
|
||||
'http://wiki.python.org/moin/PyPiXmlRpc'
|
||||
|
||||
|
||||
package = guess_package(cpv, url)
|
||||
|
||||
euscan.output.einfo("Using PyPi XMLRPC: " + package)
|
||||
@ -49,5 +50,6 @@ def scan(cpv, url):
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def brute_force(cpv, url):
|
||||
return []
|
||||
|
@ -6,9 +6,11 @@ import urllib2
|
||||
from euscan import helpers
|
||||
import euscan
|
||||
|
||||
|
||||
def can_handle(cpv, url):
|
||||
return url.startswith('mirror://rubygems/')
|
||||
|
||||
|
||||
def guess_gem(cpv, url):
|
||||
match = re.search('mirror://rubygems/(.*).gem', url)
|
||||
if match:
|
||||
@ -23,12 +25,14 @@ def guess_gem(cpv, url):
|
||||
|
||||
return pkg
|
||||
|
||||
|
||||
def scan(cpv, url):
|
||||
'http://guides.rubygems.org/rubygems-org-api/#gemversion'
|
||||
|
||||
gem = guess_gem(cpv, url)
|
||||
if not gem:
|
||||
euscan.output.eerror("Can't guess gem name using %s and %s" % (cpv, url))
|
||||
euscan.output.eerror("Can't guess gem name using %s and %s" % \
|
||||
(cpv, url))
|
||||
return []
|
||||
|
||||
url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
|
||||
@ -65,5 +69,6 @@ def scan(cpv, url):
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def brute_force(cpv, url):
|
||||
return []
|
||||
|
@ -18,6 +18,7 @@ from portage import dep
|
||||
from euscan import CONFIG, BLACKLIST_VERSIONS, ROBOTS_TXT_BLACKLIST_DOMAINS
|
||||
import euscan
|
||||
|
||||
|
||||
def htop_vercmp(a, b):
|
||||
def fixver(v):
|
||||
if v in ['0.11', '0.12', '0.13']:
|
||||
@ -33,6 +34,7 @@ VERSION_CMP_PACKAGE_QUIRKS = {
|
||||
_v_end = '((-|_)(pre|p|beta|b|alpha|a|rc|r)\d*)'
|
||||
_v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))'
|
||||
|
||||
|
||||
# Stolen from g-pypi
|
||||
def gentoo_mangle_version(up_pv):
|
||||
"""Convert PV to MY_PV if needed
|
||||
@ -146,6 +148,7 @@ def gentoo_mangle_version(up_pv):
|
||||
|
||||
return pv
|
||||
|
||||
|
||||
def cast_int_components(version):
|
||||
for i, obj in enumerate(version):
|
||||
try:
|
||||
@ -154,6 +157,7 @@ def cast_int_components(version):
|
||||
pass
|
||||
return version
|
||||
|
||||
|
||||
def simple_vercmp(a, b):
|
||||
if a == b:
|
||||
return 0
|
||||
@ -173,11 +177,13 @@ def simple_vercmp(a, b):
|
||||
else:
|
||||
return 1
|
||||
|
||||
|
||||
def vercmp(package, a, b):
|
||||
if package in VERSION_CMP_PACKAGE_QUIRKS:
|
||||
return VERSION_CMP_PACKAGE_QUIRKS[package](a, b)
|
||||
return simple_vercmp(a, b)
|
||||
|
||||
|
||||
def version_is_nightly(a, b):
|
||||
a = pkg_resources.parse_version(a)
|
||||
b = pkg_resources.parse_version(b)
|
||||
@ -188,6 +194,7 @@ def version_is_nightly(a, b):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def version_blacklisted(cp, version):
|
||||
rule = None
|
||||
cpv = '%s-%s' % (cp, version)
|
||||
@ -205,6 +212,7 @@ def version_blacklisted(cp, version):
|
||||
euscan.output.einfo("%s is blacklisted by rule %s" % (cpv, bv))
|
||||
return rule is not None
|
||||
|
||||
|
||||
def version_change_end_sep(version):
|
||||
match = re.match('.*' + _v_end, version)
|
||||
if not match:
|
||||
@ -218,6 +226,7 @@ def version_change_end_sep(version):
|
||||
return None
|
||||
return version.replace(end, newend)
|
||||
|
||||
|
||||
def version_filtered(cp, base, version, vercmp=vercmp):
|
||||
if vercmp(cp, base, version) >= 0:
|
||||
return True
|
||||
@ -230,6 +239,7 @@ def version_filtered(cp, base, version, vercmp=vercmp):
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def generate_templates_vars(version):
|
||||
ret = []
|
||||
|
||||
@ -246,6 +256,7 @@ def generate_templates_vars(version):
|
||||
ret.reverse()
|
||||
return ret
|
||||
|
||||
|
||||
def template_from_url(url, version):
|
||||
prefix, chunks = url.split('://')
|
||||
chunks = chunks.split('/')
|
||||
@ -261,6 +272,7 @@ def template_from_url(url, version):
|
||||
|
||||
return prefix + "://" + "/".join(chunks)
|
||||
|
||||
|
||||
def url_from_template(url, version):
|
||||
components = split_version(version)
|
||||
|
||||
@ -270,6 +282,7 @@ def url_from_template(url, version):
|
||||
|
||||
return url
|
||||
|
||||
|
||||
# Stolen from distutils.LooseVersion
|
||||
# Used for brute force to increment the version
|
||||
def split_version(version):
|
||||
@ -282,6 +295,7 @@ def split_version(version):
|
||||
pass
|
||||
return components
|
||||
|
||||
|
||||
def join_version(components):
|
||||
version = ""
|
||||
for i in range(len(components)):
|
||||
@ -292,6 +306,7 @@ def join_version(components):
|
||||
version += "."
|
||||
return version
|
||||
|
||||
|
||||
def increment_version(components, level):
|
||||
n = len(components)
|
||||
|
||||
@ -307,6 +322,7 @@ def increment_version(components, level):
|
||||
|
||||
return components
|
||||
|
||||
|
||||
def gen_versions(components, level):
|
||||
n = len(components)
|
||||
depth = level
|
||||
@ -325,6 +341,7 @@ def gen_versions(components, level):
|
||||
|
||||
return versions
|
||||
|
||||
|
||||
def timeout_for_url(url):
|
||||
if 'sourceforge' in url:
|
||||
timeout = 15
|
||||
@ -332,13 +349,16 @@ def timeout_for_url(url):
|
||||
timeout = 5
|
||||
return timeout
|
||||
|
||||
|
||||
class HeadRequest(urllib2.Request):
|
||||
def get_method(self):
|
||||
return "HEAD"
|
||||
|
||||
|
||||
""" RobotParser cache """
|
||||
rpcache = {}
|
||||
|
||||
|
||||
def urlallowed(url):
|
||||
if CONFIG['skip-robots-txt']:
|
||||
return True
|
||||
@ -359,7 +379,7 @@ def urlallowed(url):
|
||||
baseurl = '%s://%s' % (protocol, domain)
|
||||
robotsurl = urlparse.urljoin(baseurl, 'robots.txt')
|
||||
|
||||
if rpcache.has_key(baseurl):
|
||||
if baseurl in rpcache:
|
||||
rp = rpcache[baseurl]
|
||||
else:
|
||||
from socket import setdefaulttimeout, getdefaulttimeout
|
||||
@ -379,6 +399,7 @@ def urlallowed(url):
|
||||
|
||||
return rp.can_fetch(CONFIG['user-agent'], url) if rp else False
|
||||
|
||||
|
||||
def urlopen(url, timeout=None, verb="GET"):
|
||||
if not urlallowed(url):
|
||||
euscan.output.einfo("Url '%s' blocked by robots.txt" % url)
|
||||
@ -410,6 +431,7 @@ def urlopen(url, timeout=None, verb="GET"):
|
||||
|
||||
return opener.open(request, None, timeout)
|
||||
|
||||
|
||||
def tryurl(fileurl, template):
|
||||
result = True
|
||||
|
||||
@ -429,13 +451,16 @@ def tryurl(fileurl, template):
|
||||
|
||||
headers = fp.info()
|
||||
|
||||
if 'Content-disposition' in headers and basename not in headers['Content-disposition']:
|
||||
if 'Content-disposition' in headers and \
|
||||
basename not in headers['Content-disposition']:
|
||||
result = None
|
||||
elif 'Content-Length' in headers and headers['Content-Length'] == '0':
|
||||
result = None
|
||||
elif 'Content-Type' in headers and 'text/html' in headers['Content-Type']:
|
||||
elif 'Content-Type' in headers and \
|
||||
'text/html' in headers['Content-Type']:
|
||||
result = None
|
||||
elif 'Content-Type' in headers and 'application/x-httpd-php' in headers['Content-Type']:
|
||||
elif 'Content-Type' in headers and \
|
||||
'application/x-httpd-php' in headers['Content-Type']:
|
||||
result = None
|
||||
elif fp.geturl() != fileurl:
|
||||
regex = regex_from_template(template)
|
||||
@ -443,10 +468,10 @@ def tryurl(fileurl, template):
|
||||
basename2 = os.path.basename(fp.geturl())
|
||||
|
||||
# Redirect to another (earlier?) version
|
||||
if basename != basename2 and (re.match(regex, fp.geturl()) or re.match(baseregex, basename2)):
|
||||
if basename != basename2 and (re.match(regex, fp.geturl()) or \
|
||||
re.match(baseregex, basename2)):
|
||||
result = None
|
||||
|
||||
|
||||
if result:
|
||||
result = (fp.geturl(), fp.info())
|
||||
|
||||
@ -459,6 +484,7 @@ def tryurl(fileurl, template):
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def regex_from_template(template):
|
||||
# Escape
|
||||
template = re.escape(template)
|
||||
@ -483,6 +509,7 @@ def regex_from_template(template):
|
||||
template = template + r'/?$'
|
||||
return template
|
||||
|
||||
|
||||
def basedir_from_template(template):
|
||||
idx = template.find('${')
|
||||
if idx == -1:
|
||||
@ -494,6 +521,7 @@ def basedir_from_template(template):
|
||||
|
||||
return template[0:idx]
|
||||
|
||||
|
||||
def generate_scan_paths(url):
|
||||
prefix, chunks = url.split('://')
|
||||
chunks = chunks.split('/')
|
||||
@ -511,6 +539,7 @@ def generate_scan_paths(url):
|
||||
|
||||
return steps
|
||||
|
||||
|
||||
def parse_mirror(uri):
|
||||
from random import shuffle
|
||||
|
||||
|
@ -4,10 +4,8 @@ import sys
|
||||
import portage
|
||||
|
||||
from portage.dbapi import porttree
|
||||
from portage.output import white, yellow, turquoise, green, teal, red, EOutput
|
||||
|
||||
import gentoolkit.pprinter as pp
|
||||
from gentoolkit import errors
|
||||
from gentoolkit.query import Query
|
||||
from gentoolkit.eclean.search import (port_settings)
|
||||
|
||||
@ -17,6 +15,7 @@ from euscan import helpers
|
||||
|
||||
import euscan
|
||||
|
||||
|
||||
def filter_versions(cp, versions):
|
||||
filtered = {}
|
||||
|
||||
@ -34,6 +33,7 @@ def filter_versions(cp, versions):
|
||||
|
||||
return [(cp, filtered[version], version) for version in filtered]
|
||||
|
||||
|
||||
def scan_upstream_urls(cpv, urls):
|
||||
versions = []
|
||||
|
||||
@ -72,7 +72,9 @@ def scan_upstream(query):
|
||||
)
|
||||
|
||||
if not matches:
|
||||
sys.stderr.write(pp.warn("No package matching '%s'" % pp.pkgquery(query)))
|
||||
sys.stderr.write(
|
||||
pp.warn("No package matching '%s'" % pp.pkgquery(query))
|
||||
)
|
||||
return []
|
||||
|
||||
matches = sorted(matches)
|
||||
@ -87,11 +89,15 @@ def scan_upstream(query):
|
||||
return []
|
||||
|
||||
if pkg.cp in BLACKLIST_PACKAGES:
|
||||
sys.stderr.write(pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp)))
|
||||
sys.stderr.write(
|
||||
pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp))
|
||||
)
|
||||
return []
|
||||
|
||||
if not CONFIG['quiet']:
|
||||
pp.uprint(" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name())))
|
||||
pp.uprint(
|
||||
" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name()))
|
||||
)
|
||||
pp.uprint()
|
||||
|
||||
ebuild_path = pkg.ebuild_path()
|
||||
@ -113,7 +119,9 @@ def scan_upstream(query):
|
||||
aalist = porttree._parse_uri_map(cpv, metadata)
|
||||
except Exception as e:
|
||||
sys.stderr.write(pp.warn("%s\n" % str(e)))
|
||||
sys.stderr.write(pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv)))
|
||||
sys.stderr.write(
|
||||
pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv))
|
||||
)
|
||||
return []
|
||||
|
||||
if "mirror" in portage.settings.features:
|
||||
|
4
setup.py
4
setup.py
@ -81,8 +81,8 @@ setup(
|
||||
'https://github.com/iksaif/euscan/tarball/' +
|
||||
('master' if __version__ == '9999' else ('euscan-%s' % __version__))
|
||||
),
|
||||
install_requires=['Django==1.3.1', 'django-annoying', 'South',
|
||||
'django-piston', 'matplotlib'],
|
||||
install_requires=['Django==1.4', 'django-annoying', 'South',
|
||||
'django-piston', 'matplotlib', 'BeautifulSoup'],
|
||||
package_dir={'': 'pym'},
|
||||
packages=packages,
|
||||
package_data={},
|
||||
|
Loading…
Reference in New Issue
Block a user