some refactoring, added Package manager for removing code duplicates, added helpers module, basic tests layout

euscanwww/djeuscan/api/handlers.py

@@ -8,19 +8,14 @@ from django.forms.models import model_to_dict
 from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
     VersionLog
 
+from djeuscan.helpers import xint
+
 # replace default XMLEmitter with ours
 from piston.emitters import Emitter
 from emitters import EuscanXMLEmitter
 Emitter.register('xml', EuscanXMLEmitter, 'text/xml; charset=utf-8')
 
 
-def xint(i):
-    try:
-        return int(i)
-    except:
-        return 0
-
-
 def renameFields(vqs, fields):
     ret = []
     for n in vqs:

euscanwww/djeuscan/charts.py

@@ -1,14 +1,15 @@
 import os.path
 import time
 
-from euscanwww import settings
+import rrdtool
+import pylab
 
 from django.db.models import F, Sum
+
+from euscanwww import settings
 from djeuscan.models import Package
+from djeuscan.helpers import xint
 
-import rrdtool
-
-import pylab
 
 CHARTS_ROOT = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts')
 CHARTS_URL = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts')
@@ -20,13 +21,6 @@ pylab.rcParams['xtick.labelsize'] = 8.0
 pylab.rcParams['legend.fontsize'] = 8.0
 
 
-def xint(i):
-    try:
-        return int(i)
-    except:
-        return 0
-
-
 def chart_alive(name):
     path = os.path.join(CHARTS_ROOT, name)
     if not os.path.exists(path):

euscanwww/djeuscan/helpers.py

@@ -0,0 +1,10 @@
+"""
+djeuscan.helpers
+"""
+
+
+def xint(i):
+    try:
+        return int(i)
+    except Exception:
+        return 0

euscanwww/djeuscan/managers.py

@@ -0,0 +1,56 @@
+"""
+djeuscan.managers
+"""
+
+from django.db import models
+from djeuscan.helpers import xint
+
+
+class PackageManager(models.Manager):
+    def n_packaged(self):
+        res = self.aggregate(models.Sum('n_packaged'))['n_packaged__sum']
+        return xint(res)
+
+    def n_overlay(self):
+        res = self.aggregate(models.Sum('n_overlay'))['n_overlay__sum']
+        return xint(res)
+
+    def n_versions(self):
+        res = self.aggregate(models.Sum('n_versions'))['n_versions__sum']
+        return xint(res)
+
+    def n_upstream(self):
+        return self.n_versions() - self.n_packaged() - self.n_overlay()
+
+    def categories(self):
+        return self.values('category').annotate(
+            n_packaged=models.Sum('n_packaged'),
+            n_overlay=models.Sum('n_overlay'),
+            n_versions=models.Sum('n_versions')
+        )
+
+    def herds(self):
+        # FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of
+        # 'INNER JOIN'
+        res = self.filter(herds__isnull=False)
+        res = res.values('herds__herd').annotate(
+            n_packaged=models.Sum('n_packaged'),
+            n_overlay=models.Sum('n_overlay'),
+            n_versions=models.Sum('n_versions')
+        )
+        return res
+
+    def maintainers(self):
+        res = self.filter(maintainers__isnull=False).values(
+            'maintainers__id', 'maintainers__name', 'maintainers__email'
+        )
+        res = res.annotate(
+            n_packaged=models.Sum('n_packaged'),
+            n_overlay=models.Sum('n_overlay'),
+            n_versions=models.Sum('n_versions')
+        )
+        return res
+
+    def overlays(self):
+        res = self.values('version__overlay').exclude(version__overlay='')
+        return res.distinct()

euscanwww/djeuscan/models.py

@@ -1,7 +1,13 @@
 from django.db import models
 
+from djeuscan.managers import PackageManager
+
 
 class Herd(models.Model):
+    """
+    A herd is a collection of packages
+    """
+
     herd = models.CharField(max_length=128, unique=True)
     email = models.CharField(max_length=128, blank=True, null=True)
 
@@ -12,6 +18,10 @@ class Herd(models.Model):
 
 
 class Maintainer(models.Model):
+    """
+    The person who maintains a package
+    """
+
     name = models.CharField(max_length=128)
     email = models.CharField(max_length=128, unique=True)
 
@@ -20,6 +30,10 @@ class Maintainer(models.Model):
 
 
 class Package(models.Model):
+    """
+    A portage package
+    """
+
     category = models.CharField(max_length=128)
     name = models.CharField(max_length=128)
     description = models.TextField(blank=True)
@@ -27,12 +41,12 @@ class Package(models.Model):
     herds = models.ManyToManyField(Herd, blank=True)
     maintainers = models.ManyToManyField(Maintainer, blank=True)
 
-    ' For performance, we keep pre-computed counters '
+    # For performance, we keep pre-computed counters
     n_versions = models.IntegerField(default=0)
     n_packaged = models.IntegerField(default=0)
     n_overlay = models.IntegerField(default=0)
 
-    ' And we also pre-compute last versions '
+    # And we also pre-compute last versions
     last_version_gentoo = models.ForeignKey(
         'Version', blank=True, null=True, related_name="last_version_gentoo",
         on_delete=models.SET_NULL
@@ -46,6 +60,8 @@ class Package(models.Model):
         on_delete=models.SET_NULL
     )
 
+    objects = PackageManager()
+
     def __unicode__(self):
         return '%s/%s' % (self.category, self.name)
 
@@ -54,6 +70,10 @@ class Package(models.Model):
 
 
 class Version(models.Model):
+    """
+    Version associated to a package
+    """
+
     package = models.ForeignKey(Package)
     slot = models.CharField(max_length=128)
     revision = models.CharField(max_length=128)
@@ -110,22 +130,29 @@ class EuscanResult(models.Model):
     result = models.TextField(blank=True)
 
 
-# Keep data for charts
 class Log(models.Model):
+    """
+    Model used for keeping data for charts
+    """
+
     datetime = models.DateTimeField()
 
-    ' Packages up to date in the main portage tree '
+    # Packages up to date in the main portage tree
     n_packages_gentoo = models.IntegerField(default=0)
-    ' Packages up to date in an overlay '
+
+    # Packages up to date in an overlay
     n_packages_overlay = models.IntegerField(default=0)
-    ' Packages outdated '
+
+    # Packages outdated
     n_packages_outdated = models.IntegerField(default=0)
 
-    ' Versions in the main portage tree '
+    # Versions in the main portage tree
     n_versions_gentoo = models.IntegerField(default=0)
-    ' Versions in overlays '
+
+    # Versions in overlays
     n_versions_overlay = models.IntegerField(default=0)
-    ' Upstream versions, not in the main tree or overlays '
+
+    # Upstream versions, not in the main tree or overlays
     n_versions_upstream = models.IntegerField(default=0)
 
     def __unicode__(self):

euscanwww/djeuscan/templates/euscan/index.html

@@ -18,6 +18,8 @@ This web interface allow you to browse the portage tree, and find outdated ebuil
   <li>Versions not-packaged: {{ n_upstream }}</li>
   <li>Herds: {{ n_herds }}</li>
   <li>Maintainers: {{ n_maintainers }}</li>
+  {% if last_scan %}
     <li>Last scan: {{ last_scan }} ({{ last_scan|timedelta }})</li>
+  {% endif %}
 </ul>
 {% endblock %}

euscanwww/djeuscan/tests/__init__.py

@@ -0,0 +1,2 @@
+from .models import *
+from .views import *

euscanwww/djeuscan/tests/euscan_factory.py

@@ -0,0 +1,38 @@
+import factory
+from djeuscan.models import Herd, Maintainer, Package, Version
+
+
+class HerdFactory(factory.Factory):
+    FACTORY_FOR = Herd
+
+    herd = 'Test Herd'
+    email = 'herd@testherd.com'
+
+
+class MaintainerFactory(factory.Factory):
+    FACTORY_FOR = Maintainer
+
+    herd = 'Test Maintainer'
+    email = 'maintainer@testmaintainer.com'
+
+
+class PackageFactory(factory.Factory):
+    FACTORY_FOR = Package
+
+    category = "Test Category"
+    name = "Test Package"
+    description = "This is a test package"
+    homepage = "http://testpackage.com"
+
+
+class VersionFactory(factory.Factory):
+    FACTORY_FOR = Version
+
+    package = factory.LazyAttribute(lambda a: PackageFactory())
+    slot = "1"
+    revision = "1"
+    version = "0.1"
+    packaged = True
+    overlay = "gentoo"
+    urls = "http://packageurl.com"
+    alive = True

euscanwww/djeuscan/tests/models.py

@@ -0,0 +1,22 @@
+"""
+tests for models
+"""
+
+from django.utils import unittest
+from django.db import IntegrityError
+
+from djeuscan.tests.euscan_factory import VersionFactory, PackageFactory
+
+
+class VersionModelTests(unittest.TestCase):
+    def test_creation(self):
+        package = PackageFactory.build()
+        version = VersionFactory.build(package=package)
+        self.assertEqual(version.package, package)
+
+    def test_not_allowed_creation(self):
+        package = PackageFactory.create()
+        VersionFactory.create(package=package)
+
+        with self.assertRaises(IntegrityError):
+            VersionFactory.create(package=package)

euscanwww/djeuscan/tests/views.py

@@ -0,0 +1,12 @@
+"""
+tests for models
+"""
+
+from django.utils import unittest
+from django.test.client import RequestFactory
+
+from djeuscan.views import index
+
+
+class ViewsTests(unittest.TestCase):
+    pass

euscanwww/djeuscan/urls.py

@@ -43,7 +43,7 @@ urlpatterns = patterns('djeuscan.views',
     # Global stuff
     (r'^api/', include('djeuscan.api.urls')),
 
-    (r'^$', 'index'),
+    url(r'^$', 'index', name="index"),
     url(r'^feed/$', GlobalFeed(), name='global_feed'),
     (r'^about/$', 'about'),
     (r'^about/api$', 'api'),

euscanwww/djeuscan/views.py

@@ -1,42 +1,37 @@
+""" Views """
+
 from annoying.decorators import render_to
 from django.http import Http404
 from django.shortcuts import get_object_or_404
-from django.db.models import Sum, Max
+from django.db.models import Max
 
-from models import Version, Package, Herd, Maintainer, EuscanResult, VersionLog
+from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
-from forms import WorldForm, PackagesForm
+    VersionLog
+from djeuscan.forms import WorldForm, PackagesForm
 
 import charts
 
-""" Views """
-
 
 @render_to('euscan/index.html')
 def index(request):
-    ctx = {}
+    context = {
-    ctx['n_packaged'] = charts.xint(
+        'n_packaged': Package.objects.n_packaged(),
-        Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum']
+        'n_overlay': Package.objects.n_overlay(),
-    )
+        'n_versions': Package.objects.n_versions(),
-    ctx['n_overlay'] = charts.xint(
+        'n_upstream': Package.objects.n_upstream(),
-        Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum']
+        'n_packages': Package.objects.count(),
-    )
+        'n_herds': Herd.objects.count(),
-    ctx['n_versions'] = charts.xint(
+        'n_maintainers': Maintainer.objects.count(),
-        Package.objects.aggregate(Sum('n_versions'))['n_versions__sum']
+    }
-    )
-    ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - \
-                        ctx['n_overlay']
-    ctx['n_packages'] = Package.objects.count()
-    ctx['n_herds'] = Herd.objects.count()
-    ctx['n_maintainers'] = Maintainer.objects.count()
-
     try:
-        ctx['last_scan'] = EuscanResult.objects.get(
+        context['last_scan'] = \
+            EuscanResult.objects.get(
                 id=EuscanResult.objects.aggregate(Max('id'))['id__max']
             ).datetime
     except EuscanResult.DoesNotExist:
-        ctx['last_scan'] = None
+        context['last_scan'] = None
 
-    return ctx
+    return context
 
 
 @render_to('euscan/logs.html')
@@ -46,13 +41,7 @@ def logs(request):
 
 @render_to('euscan/categories.html')
 def categories(request):
-    categories = Package.objects.values('category').annotate(
+    return {'categories': Package.objects.categories()}
-        n_packaged=Sum('n_packaged'),
-        n_overlay=Sum('n_overlay'),
-        n_versions=Sum('n_versions')
-    )
-
-    return {'categories': categories}
 
 
 @render_to('euscan/category.html')
@@ -61,7 +50,7 @@ def category(request, category):
     packages = packages.select_related(
         'last_version_gentoo', 'last_version_overlay', 'last_version_upstream'
     )
-    print dir(packages[0])
+
     if not packages:
         raise Http404
     return {'category': category, 'packages': packages}
@@ -69,13 +58,7 @@ def category(request, category):
 
 @render_to('euscan/herds.html')
 def herds(request):
-    # FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of
+    herds = Package.objects.herds()
-    # 'INNER JOIN'
-    herds = Package.objects.filter(herds__isnull=False)
-    herds = herds.values('herds__herd').annotate(
-        n_packaged=Sum('n_packaged'),
-        n_overlay=Sum('n_overlay'),
-        n_versions=Sum('n_versions'))
     return {'herds': herds}
 
 
@@ -91,16 +74,7 @@ def herd(request, herd):
 
 @render_to('euscan/maintainers.html')
 def maintainers(request):
-    maintainers = Package.objects.filter(maintainers__isnull=False)
+    maintainers = Package.objects.maintainers()
-    maintainers = maintainers.values(
-        'maintainers__id', 'maintainers__name', 'maintainers__email'
-    )
-    maintainers = maintainers.annotate(
-        n_packaged=Sum('n_packaged'),
-        n_overlay=Sum('n_overlay'),
-        n_versions=Sum('n_versions')
-    )
-
     return {'maintainers': maintainers}
 
 
@@ -116,9 +90,7 @@ def maintainer(request, maintainer_id):
 
 @render_to('euscan/overlays.html')
 def overlays(request):
-    overlays = Package.objects.values('version__overlay')
+    overlays = Package.objects.overlays()
-    overlays = overlays.exclude(version__overlay='')
-    overlays = overlays.distinct()
     return {'overlays': overlays}
 
 

euscanwww/euscanwww/settings.py

@@ -132,15 +132,15 @@ MIDDLEWARE_CLASSES = (
     # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
 )
 
-CACHE_MIDDLEWARE_SECONDS=3600
+CACHE_MIDDLEWARE_SECONDS = 3600
-CACHE_MIDDLEWARE_ANONYMOUS_ONLY=True
+CACHE_MIDDLEWARE_ANONYMOUS_ONLY = True
 
 ROOT_URLCONF = 'euscanwww.urls'
 
 # Python dotted path to the WSGI application used by Django's runserver.
 WSGI_APPLICATION = 'euscanwww.wsgi.application'
 
-FORCE_SCRIPT_NAME=""
+FORCE_SCRIPT_NAME = ""
 
 TEMPLATE_DIRS = (
     os.path.join(SITE_ROOT, 'templates'),
@@ -166,6 +166,7 @@ INSTALLED_APPS = (
     # Uncomment the next line to enable admin documentation:
     # 'django.contrib.admindocs',
     'south',
+    'euscanwww',
     'djeuscan',
 )
 

euscanwww/euscanwww/tests.py

@@ -0,0 +1,45 @@
+"""
+System tests for euscanwww
+"""
+
+from urllib import urlencode
+
+from django.utils import unittest
+from django.test.client import Client
+from django.core.urlresolvers import reverse
+
+
+class SystemTestCase(unittest.TestCase):
+    """
+    Base class for system tests
+    """
+
+    def setUp(self):
+        self.client = Client()
+
+    def get(self, url_name, *args, **kwargs):
+        param = kwargs.pop("param", None)
+        if param:
+            url = "%s?%s" % (reverse(url_name, args=args, kwargs=kwargs),
+                             urlencode(param))
+        else:
+            url = reverse(url_name, args=args, kwargs=kwargs)
+        return self.client.get(url)
+
+    def post(self, url_name, *args, **kwargs):
+        data = kwargs.pop("data", {})
+        url = reverse(url_name, args=args, kwargs=kwargs)
+        return self.client.post(url, data)
+
+
+class NavigationTest(SystemTestCase):
+    """
+    Test main pages
+    """
+
+    def test_index(self):
+        """
+        Test index
+        """
+        response = self.get("index")
+        self.assertEqual(response.status_code, 200)