some refactoring, added Package manager for removing code duplicates, added helpers module, basic tests layout

This commit is contained in:
volpino 2012-05-01 16:56:09 +02:00
parent 1c53c60eed
commit 8e37f6249c
34 changed files with 260 additions and 84 deletions

View File

@ -8,19 +8,14 @@ from django.forms.models import model_to_dict
from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \ from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
VersionLog VersionLog
from djeuscan.helpers import xint
# replace default XMLEmitter with ours # replace default XMLEmitter with ours
from piston.emitters import Emitter from piston.emitters import Emitter
from emitters import EuscanXMLEmitter from emitters import EuscanXMLEmitter
Emitter.register('xml', EuscanXMLEmitter, 'text/xml; charset=utf-8') Emitter.register('xml', EuscanXMLEmitter, 'text/xml; charset=utf-8')
def xint(i):
try:
return int(i)
except:
return 0
def renameFields(vqs, fields): def renameFields(vqs, fields):
ret = [] ret = []
for n in vqs: for n in vqs:

View File

@ -1,14 +1,15 @@
import os.path import os.path
import time import time
from euscanwww import settings import rrdtool
import pylab
from django.db.models import F, Sum from django.db.models import F, Sum
from euscanwww import settings
from djeuscan.models import Package from djeuscan.models import Package
from djeuscan.helpers import xint
import rrdtool
import pylab
CHARTS_ROOT = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts') CHARTS_ROOT = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts')
CHARTS_URL = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts') CHARTS_URL = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts')
@ -20,13 +21,6 @@ pylab.rcParams['xtick.labelsize'] = 8.0
pylab.rcParams['legend.fontsize'] = 8.0 pylab.rcParams['legend.fontsize'] = 8.0
def xint(i):
try:
return int(i)
except:
return 0
def chart_alive(name): def chart_alive(name):
path = os.path.join(CHARTS_ROOT, name) path = os.path.join(CHARTS_ROOT, name)
if not os.path.exists(path): if not os.path.exists(path):

View File

@ -0,0 +1,10 @@
"""
djeuscan.helpers
"""
def xint(i):
try:
return int(i)
except Exception:
return 0

View File

@ -0,0 +1,56 @@
"""
djeuscan.managers
"""
from django.db import models
from djeuscan.helpers import xint
class PackageManager(models.Manager):
def n_packaged(self):
res = self.aggregate(models.Sum('n_packaged'))['n_packaged__sum']
return xint(res)
def n_overlay(self):
res = self.aggregate(models.Sum('n_overlay'))['n_overlay__sum']
return xint(res)
def n_versions(self):
res = self.aggregate(models.Sum('n_versions'))['n_versions__sum']
return xint(res)
def n_upstream(self):
return self.n_versions() - self.n_packaged() - self.n_overlay()
def categories(self):
return self.values('category').annotate(
n_packaged=models.Sum('n_packaged'),
n_overlay=models.Sum('n_overlay'),
n_versions=models.Sum('n_versions')
)
def herds(self):
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of
# 'INNER JOIN'
res = self.filter(herds__isnull=False)
res = res.values('herds__herd').annotate(
n_packaged=models.Sum('n_packaged'),
n_overlay=models.Sum('n_overlay'),
n_versions=models.Sum('n_versions')
)
return res
def maintainers(self):
res = self.filter(maintainers__isnull=False).values(
'maintainers__id', 'maintainers__name', 'maintainers__email'
)
res = res.annotate(
n_packaged=models.Sum('n_packaged'),
n_overlay=models.Sum('n_overlay'),
n_versions=models.Sum('n_versions')
)
return res
def overlays(self):
res = self.values('version__overlay').exclude(version__overlay='')
return res.distinct()

View File

@ -1,7 +1,13 @@
from django.db import models from django.db import models
from djeuscan.managers import PackageManager
class Herd(models.Model): class Herd(models.Model):
"""
A herd is a collection of packages
"""
herd = models.CharField(max_length=128, unique=True) herd = models.CharField(max_length=128, unique=True)
email = models.CharField(max_length=128, blank=True, null=True) email = models.CharField(max_length=128, blank=True, null=True)
@ -12,6 +18,10 @@ class Herd(models.Model):
class Maintainer(models.Model): class Maintainer(models.Model):
"""
The person who maintains a package
"""
name = models.CharField(max_length=128) name = models.CharField(max_length=128)
email = models.CharField(max_length=128, unique=True) email = models.CharField(max_length=128, unique=True)
@ -20,6 +30,10 @@ class Maintainer(models.Model):
class Package(models.Model): class Package(models.Model):
"""
A portage package
"""
category = models.CharField(max_length=128) category = models.CharField(max_length=128)
name = models.CharField(max_length=128) name = models.CharField(max_length=128)
description = models.TextField(blank=True) description = models.TextField(blank=True)
@ -27,12 +41,12 @@ class Package(models.Model):
herds = models.ManyToManyField(Herd, blank=True) herds = models.ManyToManyField(Herd, blank=True)
maintainers = models.ManyToManyField(Maintainer, blank=True) maintainers = models.ManyToManyField(Maintainer, blank=True)
' For performance, we keep pre-computed counters ' # For performance, we keep pre-computed counters
n_versions = models.IntegerField(default=0) n_versions = models.IntegerField(default=0)
n_packaged = models.IntegerField(default=0) n_packaged = models.IntegerField(default=0)
n_overlay = models.IntegerField(default=0) n_overlay = models.IntegerField(default=0)
' And we also pre-compute last versions ' # And we also pre-compute last versions
last_version_gentoo = models.ForeignKey( last_version_gentoo = models.ForeignKey(
'Version', blank=True, null=True, related_name="last_version_gentoo", 'Version', blank=True, null=True, related_name="last_version_gentoo",
on_delete=models.SET_NULL on_delete=models.SET_NULL
@ -46,6 +60,8 @@ class Package(models.Model):
on_delete=models.SET_NULL on_delete=models.SET_NULL
) )
objects = PackageManager()
def __unicode__(self): def __unicode__(self):
return '%s/%s' % (self.category, self.name) return '%s/%s' % (self.category, self.name)
@ -54,6 +70,10 @@ class Package(models.Model):
class Version(models.Model): class Version(models.Model):
"""
Version associated to a package
"""
package = models.ForeignKey(Package) package = models.ForeignKey(Package)
slot = models.CharField(max_length=128) slot = models.CharField(max_length=128)
revision = models.CharField(max_length=128) revision = models.CharField(max_length=128)
@ -110,22 +130,29 @@ class EuscanResult(models.Model):
result = models.TextField(blank=True) result = models.TextField(blank=True)
# Keep data for charts
class Log(models.Model): class Log(models.Model):
"""
Model used for keeping data for charts
"""
datetime = models.DateTimeField() datetime = models.DateTimeField()
' Packages up to date in the main portage tree ' # Packages up to date in the main portage tree
n_packages_gentoo = models.IntegerField(default=0) n_packages_gentoo = models.IntegerField(default=0)
' Packages up to date in an overlay '
# Packages up to date in an overlay
n_packages_overlay = models.IntegerField(default=0) n_packages_overlay = models.IntegerField(default=0)
' Packages outdated '
# Packages outdated
n_packages_outdated = models.IntegerField(default=0) n_packages_outdated = models.IntegerField(default=0)
' Versions in the main portage tree ' # Versions in the main portage tree
n_versions_gentoo = models.IntegerField(default=0) n_versions_gentoo = models.IntegerField(default=0)
' Versions in overlays '
# Versions in overlays
n_versions_overlay = models.IntegerField(default=0) n_versions_overlay = models.IntegerField(default=0)
' Upstream versions, not in the main tree or overlays '
# Upstream versions, not in the main tree or overlays
n_versions_upstream = models.IntegerField(default=0) n_versions_upstream = models.IntegerField(default=0)
def __unicode__(self): def __unicode__(self):

View File

@ -18,6 +18,8 @@ This web interface allow you to browse the portage tree, and find outdated ebuil
<li>Versions not-packaged: {{ n_upstream }}</li> <li>Versions not-packaged: {{ n_upstream }}</li>
<li>Herds: {{ n_herds }}</li> <li>Herds: {{ n_herds }}</li>
<li>Maintainers: {{ n_maintainers }}</li> <li>Maintainers: {{ n_maintainers }}</li>
{% if last_scan %}
<li>Last scan: {{ last_scan }} ({{ last_scan|timedelta }})</li> <li>Last scan: {{ last_scan }} ({{ last_scan|timedelta }})</li>
{% endif %}
</ul> </ul>
{% endblock %} {% endblock %}

View File

@ -0,0 +1,2 @@
from .models import *
from .views import *

View File

@ -0,0 +1,38 @@
import factory
from djeuscan.models import Herd, Maintainer, Package, Version
class HerdFactory(factory.Factory):
FACTORY_FOR = Herd
herd = 'Test Herd'
email = 'herd@testherd.com'
class MaintainerFactory(factory.Factory):
FACTORY_FOR = Maintainer
herd = 'Test Maintainer'
email = 'maintainer@testmaintainer.com'
class PackageFactory(factory.Factory):
FACTORY_FOR = Package
category = "Test Category"
name = "Test Package"
description = "This is a test package"
homepage = "http://testpackage.com"
class VersionFactory(factory.Factory):
FACTORY_FOR = Version
package = factory.LazyAttribute(lambda a: PackageFactory())
slot = "1"
revision = "1"
version = "0.1"
packaged = True
overlay = "gentoo"
urls = "http://packageurl.com"
alive = True

View File

@ -0,0 +1,22 @@
"""
tests for models
"""
from django.utils import unittest
from django.db import IntegrityError
from djeuscan.tests.euscan_factory import VersionFactory, PackageFactory
class VersionModelTests(unittest.TestCase):
def test_creation(self):
package = PackageFactory.build()
version = VersionFactory.build(package=package)
self.assertEqual(version.package, package)
def test_not_allowed_creation(self):
package = PackageFactory.create()
VersionFactory.create(package=package)
with self.assertRaises(IntegrityError):
VersionFactory.create(package=package)

View File

@ -0,0 +1,12 @@
"""
tests for models
"""
from django.utils import unittest
from django.test.client import RequestFactory
from djeuscan.views import index
class ViewsTests(unittest.TestCase):
pass

View File

@ -43,7 +43,7 @@ urlpatterns = patterns('djeuscan.views',
# Global stuff # Global stuff
(r'^api/', include('djeuscan.api.urls')), (r'^api/', include('djeuscan.api.urls')),
(r'^$', 'index'), url(r'^$', 'index', name="index"),
url(r'^feed/$', GlobalFeed(), name='global_feed'), url(r'^feed/$', GlobalFeed(), name='global_feed'),
(r'^about/$', 'about'), (r'^about/$', 'about'),
(r'^about/api$', 'api'), (r'^about/api$', 'api'),

View File

@ -1,42 +1,37 @@
""" Views """
from annoying.decorators import render_to from annoying.decorators import render_to
from django.http import Http404 from django.http import Http404
from django.shortcuts import get_object_or_404 from django.shortcuts import get_object_or_404
from django.db.models import Sum, Max from django.db.models import Max
from models import Version, Package, Herd, Maintainer, EuscanResult, VersionLog from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
from forms import WorldForm, PackagesForm VersionLog
from djeuscan.forms import WorldForm, PackagesForm
import charts import charts
""" Views """
@render_to('euscan/index.html') @render_to('euscan/index.html')
def index(request): def index(request):
ctx = {} context = {
ctx['n_packaged'] = charts.xint( 'n_packaged': Package.objects.n_packaged(),
Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum'] 'n_overlay': Package.objects.n_overlay(),
) 'n_versions': Package.objects.n_versions(),
ctx['n_overlay'] = charts.xint( 'n_upstream': Package.objects.n_upstream(),
Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum'] 'n_packages': Package.objects.count(),
) 'n_herds': Herd.objects.count(),
ctx['n_versions'] = charts.xint( 'n_maintainers': Maintainer.objects.count(),
Package.objects.aggregate(Sum('n_versions'))['n_versions__sum'] }
)
ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - \
ctx['n_overlay']
ctx['n_packages'] = Package.objects.count()
ctx['n_herds'] = Herd.objects.count()
ctx['n_maintainers'] = Maintainer.objects.count()
try: try:
ctx['last_scan'] = EuscanResult.objects.get( context['last_scan'] = \
EuscanResult.objects.get(
id=EuscanResult.objects.aggregate(Max('id'))['id__max'] id=EuscanResult.objects.aggregate(Max('id'))['id__max']
).datetime ).datetime
except EuscanResult.DoesNotExist: except EuscanResult.DoesNotExist:
ctx['last_scan'] = None context['last_scan'] = None
return ctx return context
@render_to('euscan/logs.html') @render_to('euscan/logs.html')
@ -46,13 +41,7 @@ def logs(request):
@render_to('euscan/categories.html') @render_to('euscan/categories.html')
def categories(request): def categories(request):
categories = Package.objects.values('category').annotate( return {'categories': Package.objects.categories()}
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')
)
return {'categories': categories}
@render_to('euscan/category.html') @render_to('euscan/category.html')
@ -61,7 +50,7 @@ def category(request, category):
packages = packages.select_related( packages = packages.select_related(
'last_version_gentoo', 'last_version_overlay', 'last_version_upstream' 'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
) )
print dir(packages[0])
if not packages: if not packages:
raise Http404 raise Http404
return {'category': category, 'packages': packages} return {'category': category, 'packages': packages}
@ -69,13 +58,7 @@ def category(request, category):
@render_to('euscan/herds.html') @render_to('euscan/herds.html')
def herds(request): def herds(request):
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of herds = Package.objects.herds()
# 'INNER JOIN'
herds = Package.objects.filter(herds__isnull=False)
herds = herds.values('herds__herd').annotate(
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
return {'herds': herds} return {'herds': herds}
@ -91,16 +74,7 @@ def herd(request, herd):
@render_to('euscan/maintainers.html') @render_to('euscan/maintainers.html')
def maintainers(request): def maintainers(request):
maintainers = Package.objects.filter(maintainers__isnull=False) maintainers = Package.objects.maintainers()
maintainers = maintainers.values(
'maintainers__id', 'maintainers__name', 'maintainers__email'
)
maintainers = maintainers.annotate(
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')
)
return {'maintainers': maintainers} return {'maintainers': maintainers}
@ -116,9 +90,7 @@ def maintainer(request, maintainer_id):
@render_to('euscan/overlays.html') @render_to('euscan/overlays.html')
def overlays(request): def overlays(request):
overlays = Package.objects.values('version__overlay') overlays = Package.objects.overlays()
overlays = overlays.exclude(version__overlay='')
overlays = overlays.distinct()
return {'overlays': overlays} return {'overlays': overlays}

View File

View File

@ -166,6 +166,7 @@ INSTALLED_APPS = (
# Uncomment the next line to enable admin documentation: # Uncomment the next line to enable admin documentation:
# 'django.contrib.admindocs', # 'django.contrib.admindocs',
'south', 'south',
'euscanwww',
'djeuscan', 'djeuscan',
) )

View File

@ -0,0 +1,45 @@
"""
System tests for euscanwww
"""
from urllib import urlencode
from django.utils import unittest
from django.test.client import Client
from django.core.urlresolvers import reverse
class SystemTestCase(unittest.TestCase):
"""
Base class for system tests
"""
def setUp(self):
self.client = Client()
def get(self, url_name, *args, **kwargs):
param = kwargs.pop("param", None)
if param:
url = "%s?%s" % (reverse(url_name, args=args, kwargs=kwargs),
urlencode(param))
else:
url = reverse(url_name, args=args, kwargs=kwargs)
return self.client.get(url)
def post(self, url_name, *args, **kwargs):
data = kwargs.pop("data", {})
url = reverse(url_name, args=args, kwargs=kwargs)
return self.client.post(url, data)
class NavigationTest(SystemTestCase):
"""
Test main pages
"""
def test_index(self):
"""
Test index
"""
response = self.get("index")
self.assertEqual(response.status_code, 200)