some refactoring, added Package manager for removing code duplicates, added helpers module, basic tests layout

This commit is contained in:
volpino 2012-05-01 16:56:09 +02:00
parent 1c53c60eed
commit 8e37f6249c
34 changed files with 260 additions and 84 deletions

View File

@ -8,19 +8,14 @@ from django.forms.models import model_to_dict
from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
VersionLog
from djeuscan.helpers import xint
# replace default XMLEmitter with ours
from piston.emitters import Emitter
from emitters import EuscanXMLEmitter
Emitter.register('xml', EuscanXMLEmitter, 'text/xml; charset=utf-8')
def xint(i):
try:
return int(i)
except:
return 0
def renameFields(vqs, fields):
ret = []
for n in vqs:

View File

@ -1,14 +1,15 @@
import os.path
import time
from euscanwww import settings
import rrdtool
import pylab
from django.db.models import F, Sum
from euscanwww import settings
from djeuscan.models import Package
from djeuscan.helpers import xint
import rrdtool
import pylab
CHARTS_ROOT = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts')
CHARTS_URL = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts')
@ -20,13 +21,6 @@ pylab.rcParams['xtick.labelsize'] = 8.0
pylab.rcParams['legend.fontsize'] = 8.0
def xint(i):
try:
return int(i)
except:
return 0
def chart_alive(name):
path = os.path.join(CHARTS_ROOT, name)
if not os.path.exists(path):

View File

@ -0,0 +1,10 @@
"""
djeuscan.helpers
"""
def xint(i):
try:
return int(i)
except Exception:
return 0

View File

@ -0,0 +1,56 @@
"""
djeuscan.managers
"""
from django.db import models
from djeuscan.helpers import xint
class PackageManager(models.Manager):
def n_packaged(self):
res = self.aggregate(models.Sum('n_packaged'))['n_packaged__sum']
return xint(res)
def n_overlay(self):
res = self.aggregate(models.Sum('n_overlay'))['n_overlay__sum']
return xint(res)
def n_versions(self):
res = self.aggregate(models.Sum('n_versions'))['n_versions__sum']
return xint(res)
def n_upstream(self):
return self.n_versions() - self.n_packaged() - self.n_overlay()
def categories(self):
return self.values('category').annotate(
n_packaged=models.Sum('n_packaged'),
n_overlay=models.Sum('n_overlay'),
n_versions=models.Sum('n_versions')
)
def herds(self):
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of
# 'INNER JOIN'
res = self.filter(herds__isnull=False)
res = res.values('herds__herd').annotate(
n_packaged=models.Sum('n_packaged'),
n_overlay=models.Sum('n_overlay'),
n_versions=models.Sum('n_versions')
)
return res
def maintainers(self):
res = self.filter(maintainers__isnull=False).values(
'maintainers__id', 'maintainers__name', 'maintainers__email'
)
res = res.annotate(
n_packaged=models.Sum('n_packaged'),
n_overlay=models.Sum('n_overlay'),
n_versions=models.Sum('n_versions')
)
return res
def overlays(self):
res = self.values('version__overlay').exclude(version__overlay='')
return res.distinct()

View File

@ -1,7 +1,13 @@
from django.db import models
from djeuscan.managers import PackageManager
class Herd(models.Model):
"""
A herd is a collection of packages
"""
herd = models.CharField(max_length=128, unique=True)
email = models.CharField(max_length=128, blank=True, null=True)
@ -12,6 +18,10 @@ class Herd(models.Model):
class Maintainer(models.Model):
"""
The person who maintains a package
"""
name = models.CharField(max_length=128)
email = models.CharField(max_length=128, unique=True)
@ -20,6 +30,10 @@ class Maintainer(models.Model):
class Package(models.Model):
"""
A portage package
"""
category = models.CharField(max_length=128)
name = models.CharField(max_length=128)
description = models.TextField(blank=True)
@ -27,12 +41,12 @@ class Package(models.Model):
herds = models.ManyToManyField(Herd, blank=True)
maintainers = models.ManyToManyField(Maintainer, blank=True)
' For performance, we keep pre-computed counters '
# For performance, we keep pre-computed counters
n_versions = models.IntegerField(default=0)
n_packaged = models.IntegerField(default=0)
n_overlay = models.IntegerField(default=0)
' And we also pre-compute last versions '
# And we also pre-compute last versions
last_version_gentoo = models.ForeignKey(
'Version', blank=True, null=True, related_name="last_version_gentoo",
on_delete=models.SET_NULL
@ -46,6 +60,8 @@ class Package(models.Model):
on_delete=models.SET_NULL
)
objects = PackageManager()
def __unicode__(self):
return '%s/%s' % (self.category, self.name)
@ -54,6 +70,10 @@ class Package(models.Model):
class Version(models.Model):
"""
Version associated to a package
"""
package = models.ForeignKey(Package)
slot = models.CharField(max_length=128)
revision = models.CharField(max_length=128)
@ -110,22 +130,29 @@ class EuscanResult(models.Model):
result = models.TextField(blank=True)
# Keep data for charts
class Log(models.Model):
"""
Model used for keeping data for charts
"""
datetime = models.DateTimeField()
' Packages up to date in the main portage tree '
# Packages up to date in the main portage tree
n_packages_gentoo = models.IntegerField(default=0)
' Packages up to date in an overlay '
# Packages up to date in an overlay
n_packages_overlay = models.IntegerField(default=0)
' Packages outdated '
# Packages outdated
n_packages_outdated = models.IntegerField(default=0)
' Versions in the main portage tree '
# Versions in the main portage tree
n_versions_gentoo = models.IntegerField(default=0)
' Versions in overlays '
# Versions in overlays
n_versions_overlay = models.IntegerField(default=0)
' Upstream versions, not in the main tree or overlays '
# Upstream versions, not in the main tree or overlays
n_versions_upstream = models.IntegerField(default=0)
def __unicode__(self):

View File

@ -18,6 +18,8 @@ This web interface allow you to browse the portage tree, and find outdated ebuil
<li>Versions not-packaged: {{ n_upstream }}</li>
<li>Herds: {{ n_herds }}</li>
<li>Maintainers: {{ n_maintainers }}</li>
<li>Last scan: {{ last_scan }} ({{ last_scan|timedelta }})</li>
{% if last_scan %}
<li>Last scan: {{ last_scan }} ({{ last_scan|timedelta }})</li>
{% endif %}
</ul>
{% endblock %}

View File

@ -0,0 +1,2 @@
from .models import *
from .views import *

View File

@ -0,0 +1,38 @@
import factory
from djeuscan.models import Herd, Maintainer, Package, Version
class HerdFactory(factory.Factory):
FACTORY_FOR = Herd
herd = 'Test Herd'
email = 'herd@testherd.com'
class MaintainerFactory(factory.Factory):
FACTORY_FOR = Maintainer
herd = 'Test Maintainer'
email = 'maintainer@testmaintainer.com'
class PackageFactory(factory.Factory):
FACTORY_FOR = Package
category = "Test Category"
name = "Test Package"
description = "This is a test package"
homepage = "http://testpackage.com"
class VersionFactory(factory.Factory):
FACTORY_FOR = Version
package = factory.LazyAttribute(lambda a: PackageFactory())
slot = "1"
revision = "1"
version = "0.1"
packaged = True
overlay = "gentoo"
urls = "http://packageurl.com"
alive = True

View File

@ -0,0 +1,22 @@
"""
tests for models
"""
from django.utils import unittest
from django.db import IntegrityError
from djeuscan.tests.euscan_factory import VersionFactory, PackageFactory
class VersionModelTests(unittest.TestCase):
def test_creation(self):
package = PackageFactory.build()
version = VersionFactory.build(package=package)
self.assertEqual(version.package, package)
def test_not_allowed_creation(self):
package = PackageFactory.create()
VersionFactory.create(package=package)
with self.assertRaises(IntegrityError):
VersionFactory.create(package=package)

View File

@ -0,0 +1,12 @@
"""
tests for models
"""
from django.utils import unittest
from django.test.client import RequestFactory
from djeuscan.views import index
class ViewsTests(unittest.TestCase):
pass

View File

@ -43,7 +43,7 @@ urlpatterns = patterns('djeuscan.views',
# Global stuff
(r'^api/', include('djeuscan.api.urls')),
(r'^$', 'index'),
url(r'^$', 'index', name="index"),
url(r'^feed/$', GlobalFeed(), name='global_feed'),
(r'^about/$', 'about'),
(r'^about/api$', 'api'),

View File

@ -1,42 +1,37 @@
""" Views """
from annoying.decorators import render_to
from django.http import Http404
from django.shortcuts import get_object_or_404
from django.db.models import Sum, Max
from django.db.models import Max
from models import Version, Package, Herd, Maintainer, EuscanResult, VersionLog
from forms import WorldForm, PackagesForm
from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
VersionLog
from djeuscan.forms import WorldForm, PackagesForm
import charts
""" Views """
@render_to('euscan/index.html')
def index(request):
ctx = {}
ctx['n_packaged'] = charts.xint(
Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum']
)
ctx['n_overlay'] = charts.xint(
Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum']
)
ctx['n_versions'] = charts.xint(
Package.objects.aggregate(Sum('n_versions'))['n_versions__sum']
)
ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - \
ctx['n_overlay']
ctx['n_packages'] = Package.objects.count()
ctx['n_herds'] = Herd.objects.count()
ctx['n_maintainers'] = Maintainer.objects.count()
context = {
'n_packaged': Package.objects.n_packaged(),
'n_overlay': Package.objects.n_overlay(),
'n_versions': Package.objects.n_versions(),
'n_upstream': Package.objects.n_upstream(),
'n_packages': Package.objects.count(),
'n_herds': Herd.objects.count(),
'n_maintainers': Maintainer.objects.count(),
}
try:
ctx['last_scan'] = EuscanResult.objects.get(
context['last_scan'] = \
EuscanResult.objects.get(
id=EuscanResult.objects.aggregate(Max('id'))['id__max']
).datetime
except EuscanResult.DoesNotExist:
ctx['last_scan'] = None
context['last_scan'] = None
return ctx
return context
@render_to('euscan/logs.html')
@ -46,13 +41,7 @@ def logs(request):
@render_to('euscan/categories.html')
def categories(request):
categories = Package.objects.values('category').annotate(
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')
)
return {'categories': categories}
return {'categories': Package.objects.categories()}
@render_to('euscan/category.html')
@ -61,7 +50,7 @@ def category(request, category):
packages = packages.select_related(
'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
)
print dir(packages[0])
if not packages:
raise Http404
return {'category': category, 'packages': packages}
@ -69,13 +58,7 @@ def category(request, category):
@render_to('euscan/herds.html')
def herds(request):
# FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of
# 'INNER JOIN'
herds = Package.objects.filter(herds__isnull=False)
herds = herds.values('herds__herd').annotate(
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions'))
herds = Package.objects.herds()
return {'herds': herds}
@ -91,16 +74,7 @@ def herd(request, herd):
@render_to('euscan/maintainers.html')
def maintainers(request):
maintainers = Package.objects.filter(maintainers__isnull=False)
maintainers = maintainers.values(
'maintainers__id', 'maintainers__name', 'maintainers__email'
)
maintainers = maintainers.annotate(
n_packaged=Sum('n_packaged'),
n_overlay=Sum('n_overlay'),
n_versions=Sum('n_versions')
)
maintainers = Package.objects.maintainers()
return {'maintainers': maintainers}
@ -116,9 +90,7 @@ def maintainer(request, maintainer_id):
@render_to('euscan/overlays.html')
def overlays(request):
overlays = Package.objects.values('version__overlay')
overlays = overlays.exclude(version__overlay='')
overlays = overlays.distinct()
overlays = Package.objects.overlays()
return {'overlays': overlays}

View File

View File

@ -132,15 +132,15 @@ MIDDLEWARE_CLASSES = (
# 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)
CACHE_MIDDLEWARE_SECONDS=3600
CACHE_MIDDLEWARE_ANONYMOUS_ONLY=True
CACHE_MIDDLEWARE_SECONDS = 3600
CACHE_MIDDLEWARE_ANONYMOUS_ONLY = True
ROOT_URLCONF = 'euscanwww.urls'
# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'euscanwww.wsgi.application'
FORCE_SCRIPT_NAME=""
FORCE_SCRIPT_NAME = ""
TEMPLATE_DIRS = (
os.path.join(SITE_ROOT, 'templates'),
@ -166,6 +166,7 @@ INSTALLED_APPS = (
# Uncomment the next line to enable admin documentation:
# 'django.contrib.admindocs',
'south',
'euscanwww',
'djeuscan',
)

View File

@ -0,0 +1,45 @@
"""
System tests for euscanwww
"""
from urllib import urlencode
from django.utils import unittest
from django.test.client import Client
from django.core.urlresolvers import reverse
class SystemTestCase(unittest.TestCase):
"""
Base class for system tests
"""
def setUp(self):
self.client = Client()
def get(self, url_name, *args, **kwargs):
param = kwargs.pop("param", None)
if param:
url = "%s?%s" % (reverse(url_name, args=args, kwargs=kwargs),
urlencode(param))
else:
url = reverse(url_name, args=args, kwargs=kwargs)
return self.client.get(url)
def post(self, url_name, *args, **kwargs):
data = kwargs.pop("data", {})
url = reverse(url_name, args=args, kwargs=kwargs)
return self.client.post(url, data)
class NavigationTest(SystemTestCase):
"""
Test main pages
"""
def test_index(self):
"""
Test index
"""
response = self.get("index")
self.assertEqual(response.status_code, 200)