diff --git a/euscanwww/djeuscan/api/handlers.py b/euscanwww/djeuscan/api/handlers.py index 2cfa53a..047bf4e 100644 --- a/euscanwww/djeuscan/api/handlers.py +++ b/euscanwww/djeuscan/api/handlers.py @@ -8,19 +8,14 @@ from django.forms.models import model_to_dict from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \ VersionLog +from djeuscan.helpers import xint + # replace default XMLEmitter with ours from piston.emitters import Emitter from emitters import EuscanXMLEmitter Emitter.register('xml', EuscanXMLEmitter, 'text/xml; charset=utf-8') -def xint(i): - try: - return int(i) - except: - return 0 - - def renameFields(vqs, fields): ret = [] for n in vqs: diff --git a/euscanwww/djeuscan/charts.py b/euscanwww/djeuscan/charts.py index 276e914..78239dc 100644 --- a/euscanwww/djeuscan/charts.py +++ b/euscanwww/djeuscan/charts.py @@ -1,14 +1,15 @@ import os.path import time -from euscanwww import settings +import rrdtool +import pylab from django.db.models import F, Sum + +from euscanwww import settings from djeuscan.models import Package +from djeuscan.helpers import xint -import rrdtool - -import pylab CHARTS_ROOT = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts') CHARTS_URL = os.path.join(settings.EUSCAN_ROOT, 'var', 'charts') @@ -20,13 +21,6 @@ pylab.rcParams['xtick.labelsize'] = 8.0 pylab.rcParams['legend.fontsize'] = 8.0 -def xint(i): - try: - return int(i) - except: - return 0 - - def chart_alive(name): path = os.path.join(CHARTS_ROOT, name) if not os.path.exists(path): diff --git a/euscanwww/djeuscan/helpers.py b/euscanwww/djeuscan/helpers.py new file mode 100644 index 0000000..f771159 --- /dev/null +++ b/euscanwww/djeuscan/helpers.py @@ -0,0 +1,10 @@ +""" +djeuscan.helpers +""" + + +def xint(i): + try: + return int(i) + except Exception: + return 0 diff --git a/euscanwww/djeuscan/managers.py b/euscanwww/djeuscan/managers.py new file mode 100644 index 0000000..f9f6156 --- /dev/null +++ b/euscanwww/djeuscan/managers.py @@ -0,0 +1,56 @@ +""" +djeuscan.managers +""" + +from django.db import models +from djeuscan.helpers import xint + + +class PackageManager(models.Manager): + def n_packaged(self): + res = self.aggregate(models.Sum('n_packaged'))['n_packaged__sum'] + return xint(res) + + def n_overlay(self): + res = self.aggregate(models.Sum('n_overlay'))['n_overlay__sum'] + return xint(res) + + def n_versions(self): + res = self.aggregate(models.Sum('n_versions'))['n_versions__sum'] + return xint(res) + + def n_upstream(self): + return self.n_versions() - self.n_packaged() - self.n_overlay() + + def categories(self): + return self.values('category').annotate( + n_packaged=models.Sum('n_packaged'), + n_overlay=models.Sum('n_overlay'), + n_versions=models.Sum('n_versions') + ) + + def herds(self): + # FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of + # 'INNER JOIN' + res = self.filter(herds__isnull=False) + res = res.values('herds__herd').annotate( + n_packaged=models.Sum('n_packaged'), + n_overlay=models.Sum('n_overlay'), + n_versions=models.Sum('n_versions') + ) + return res + + def maintainers(self): + res = self.filter(maintainers__isnull=False).values( + 'maintainers__id', 'maintainers__name', 'maintainers__email' + ) + res = res.annotate( + n_packaged=models.Sum('n_packaged'), + n_overlay=models.Sum('n_overlay'), + n_versions=models.Sum('n_versions') + ) + return res + + def overlays(self): + res = self.values('version__overlay').exclude(version__overlay='') + return res.distinct() diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py index 8555c2e..5c9b7d0 100644 --- a/euscanwww/djeuscan/models.py +++ b/euscanwww/djeuscan/models.py @@ -1,7 +1,13 @@ from django.db import models +from djeuscan.managers import PackageManager + class Herd(models.Model): + """ + A herd is a collection of packages + """ + herd = models.CharField(max_length=128, unique=True) email = models.CharField(max_length=128, blank=True, null=True) @@ -12,6 +18,10 @@ class Herd(models.Model): class Maintainer(models.Model): + """ + The person who maintains a package + """ + name = models.CharField(max_length=128) email = models.CharField(max_length=128, unique=True) @@ -20,6 +30,10 @@ class Maintainer(models.Model): class Package(models.Model): + """ + A portage package + """ + category = models.CharField(max_length=128) name = models.CharField(max_length=128) description = models.TextField(blank=True) @@ -27,12 +41,12 @@ class Package(models.Model): herds = models.ManyToManyField(Herd, blank=True) maintainers = models.ManyToManyField(Maintainer, blank=True) - ' For performance, we keep pre-computed counters ' + # For performance, we keep pre-computed counters n_versions = models.IntegerField(default=0) n_packaged = models.IntegerField(default=0) n_overlay = models.IntegerField(default=0) - ' And we also pre-compute last versions ' + # And we also pre-compute last versions last_version_gentoo = models.ForeignKey( 'Version', blank=True, null=True, related_name="last_version_gentoo", on_delete=models.SET_NULL @@ -46,6 +60,8 @@ class Package(models.Model): on_delete=models.SET_NULL ) + objects = PackageManager() + def __unicode__(self): return '%s/%s' % (self.category, self.name) @@ -54,6 +70,10 @@ class Package(models.Model): class Version(models.Model): + """ + Version associated to a package + """ + package = models.ForeignKey(Package) slot = models.CharField(max_length=128) revision = models.CharField(max_length=128) @@ -110,22 +130,29 @@ class EuscanResult(models.Model): result = models.TextField(blank=True) -# Keep data for charts class Log(models.Model): + """ + Model used for keeping data for charts + """ + datetime = models.DateTimeField() - ' Packages up to date in the main portage tree ' + # Packages up to date in the main portage tree n_packages_gentoo = models.IntegerField(default=0) - ' Packages up to date in an overlay ' + + # Packages up to date in an overlay n_packages_overlay = models.IntegerField(default=0) - ' Packages outdated ' + + # Packages outdated n_packages_outdated = models.IntegerField(default=0) - ' Versions in the main portage tree ' + # Versions in the main portage tree n_versions_gentoo = models.IntegerField(default=0) - ' Versions in overlays ' + + # Versions in overlays n_versions_overlay = models.IntegerField(default=0) - ' Upstream versions, not in the main tree or overlays ' + + # Upstream versions, not in the main tree or overlays n_versions_upstream = models.IntegerField(default=0) def __unicode__(self): diff --git a/euscanwww/templates/_base.html b/euscanwww/djeuscan/templates/_base.html similarity index 100% rename from euscanwww/templates/_base.html rename to euscanwww/djeuscan/templates/_base.html diff --git a/euscanwww/templates/euscan/_datatable.html b/euscanwww/djeuscan/templates/euscan/_datatable.html similarity index 100% rename from euscanwww/templates/euscan/_datatable.html rename to euscanwww/djeuscan/templates/euscan/_datatable.html diff --git a/euscanwww/templates/euscan/_package_bar.html b/euscanwww/djeuscan/templates/euscan/_package_bar.html similarity index 100% rename from euscanwww/templates/euscan/_package_bar.html rename to euscanwww/djeuscan/templates/euscan/_package_bar.html diff --git a/euscanwww/templates/euscan/_package_cols.html b/euscanwww/djeuscan/templates/euscan/_package_cols.html similarity index 100% rename from euscanwww/templates/euscan/_package_cols.html rename to euscanwww/djeuscan/templates/euscan/_package_cols.html diff --git a/euscanwww/templates/euscan/_packages.html b/euscanwww/djeuscan/templates/euscan/_packages.html similarity index 100% rename from euscanwww/templates/euscan/_packages.html rename to euscanwww/djeuscan/templates/euscan/_packages.html diff --git a/euscanwww/templates/euscan/about.html b/euscanwww/djeuscan/templates/euscan/about.html similarity index 100% rename from euscanwww/templates/euscan/about.html rename to euscanwww/djeuscan/templates/euscan/about.html diff --git a/euscanwww/templates/euscan/api.html b/euscanwww/djeuscan/templates/euscan/api.html similarity index 100% rename from euscanwww/templates/euscan/api.html rename to euscanwww/djeuscan/templates/euscan/api.html diff --git a/euscanwww/templates/euscan/categories.html b/euscanwww/djeuscan/templates/euscan/categories.html similarity index 100% rename from euscanwww/templates/euscan/categories.html rename to euscanwww/djeuscan/templates/euscan/categories.html diff --git a/euscanwww/templates/euscan/category.html b/euscanwww/djeuscan/templates/euscan/category.html similarity index 100% rename from euscanwww/templates/euscan/category.html rename to euscanwww/djeuscan/templates/euscan/category.html diff --git a/euscanwww/templates/euscan/herd.html b/euscanwww/djeuscan/templates/euscan/herd.html similarity index 100% rename from euscanwww/templates/euscan/herd.html rename to euscanwww/djeuscan/templates/euscan/herd.html diff --git a/euscanwww/templates/euscan/herds.html b/euscanwww/djeuscan/templates/euscan/herds.html similarity index 100% rename from euscanwww/templates/euscan/herds.html rename to euscanwww/djeuscan/templates/euscan/herds.html diff --git a/euscanwww/templates/euscan/index.html b/euscanwww/djeuscan/templates/euscan/index.html similarity index 85% rename from euscanwww/templates/euscan/index.html rename to euscanwww/djeuscan/templates/euscan/index.html index 1a2f471..989690b 100644 --- a/euscanwww/templates/euscan/index.html +++ b/euscanwww/djeuscan/templates/euscan/index.html @@ -18,6 +18,8 @@ This web interface allow you to browse the portage tree, and find outdated ebuil
  • Versions not-packaged: {{ n_upstream }}
  • Herds: {{ n_herds }}
  • Maintainers: {{ n_maintainers }}
  • -
  • Last scan: {{ last_scan }} ({{ last_scan|timedelta }})
  • + {% if last_scan %} +
  • Last scan: {{ last_scan }} ({{ last_scan|timedelta }})
  • + {% endif %} {% endblock %} diff --git a/euscanwww/templates/euscan/maintainer.html b/euscanwww/djeuscan/templates/euscan/maintainer.html similarity index 100% rename from euscanwww/templates/euscan/maintainer.html rename to euscanwww/djeuscan/templates/euscan/maintainer.html diff --git a/euscanwww/templates/euscan/maintainers.html b/euscanwww/djeuscan/templates/euscan/maintainers.html similarity index 100% rename from euscanwww/templates/euscan/maintainers.html rename to euscanwww/djeuscan/templates/euscan/maintainers.html diff --git a/euscanwww/templates/euscan/overlay.html b/euscanwww/djeuscan/templates/euscan/overlay.html similarity index 100% rename from euscanwww/templates/euscan/overlay.html rename to euscanwww/djeuscan/templates/euscan/overlay.html diff --git a/euscanwww/templates/euscan/overlays.html b/euscanwww/djeuscan/templates/euscan/overlays.html similarity index 100% rename from euscanwww/templates/euscan/overlays.html rename to euscanwww/djeuscan/templates/euscan/overlays.html diff --git a/euscanwww/templates/euscan/package.html b/euscanwww/djeuscan/templates/euscan/package.html similarity index 100% rename from euscanwww/templates/euscan/package.html rename to euscanwww/djeuscan/templates/euscan/package.html diff --git a/euscanwww/templates/euscan/statistics.html b/euscanwww/djeuscan/templates/euscan/statistics.html similarity index 100% rename from euscanwww/templates/euscan/statistics.html rename to euscanwww/djeuscan/templates/euscan/statistics.html diff --git a/euscanwww/templates/euscan/world.html b/euscanwww/djeuscan/templates/euscan/world.html similarity index 100% rename from euscanwww/templates/euscan/world.html rename to euscanwww/djeuscan/templates/euscan/world.html diff --git a/euscanwww/templates/euscan/world_scan.html b/euscanwww/djeuscan/templates/euscan/world_scan.html similarity index 100% rename from euscanwww/templates/euscan/world_scan.html rename to euscanwww/djeuscan/templates/euscan/world_scan.html diff --git a/euscanwww/djeuscan/tests/__init__.py b/euscanwww/djeuscan/tests/__init__.py new file mode 100644 index 0000000..378409d --- /dev/null +++ b/euscanwww/djeuscan/tests/__init__.py @@ -0,0 +1,2 @@ +from .models import * +from .views import * diff --git a/euscanwww/djeuscan/tests/euscan_factory.py b/euscanwww/djeuscan/tests/euscan_factory.py new file mode 100644 index 0000000..476b9d1 --- /dev/null +++ b/euscanwww/djeuscan/tests/euscan_factory.py @@ -0,0 +1,38 @@ +import factory +from djeuscan.models import Herd, Maintainer, Package, Version + + +class HerdFactory(factory.Factory): + FACTORY_FOR = Herd + + herd = 'Test Herd' + email = 'herd@testherd.com' + + +class MaintainerFactory(factory.Factory): + FACTORY_FOR = Maintainer + + herd = 'Test Maintainer' + email = 'maintainer@testmaintainer.com' + + +class PackageFactory(factory.Factory): + FACTORY_FOR = Package + + category = "Test Category" + name = "Test Package" + description = "This is a test package" + homepage = "http://testpackage.com" + + +class VersionFactory(factory.Factory): + FACTORY_FOR = Version + + package = factory.LazyAttribute(lambda a: PackageFactory()) + slot = "1" + revision = "1" + version = "0.1" + packaged = True + overlay = "gentoo" + urls = "http://packageurl.com" + alive = True diff --git a/euscanwww/djeuscan/tests/models.py b/euscanwww/djeuscan/tests/models.py new file mode 100644 index 0000000..5c9dd26 --- /dev/null +++ b/euscanwww/djeuscan/tests/models.py @@ -0,0 +1,22 @@ +""" +tests for models +""" + +from django.utils import unittest +from django.db import IntegrityError + +from djeuscan.tests.euscan_factory import VersionFactory, PackageFactory + + +class VersionModelTests(unittest.TestCase): + def test_creation(self): + package = PackageFactory.build() + version = VersionFactory.build(package=package) + self.assertEqual(version.package, package) + + def test_not_allowed_creation(self): + package = PackageFactory.create() + VersionFactory.create(package=package) + + with self.assertRaises(IntegrityError): + VersionFactory.create(package=package) diff --git a/euscanwww/djeuscan/tests/views.py b/euscanwww/djeuscan/tests/views.py new file mode 100644 index 0000000..89ae0ae --- /dev/null +++ b/euscanwww/djeuscan/tests/views.py @@ -0,0 +1,12 @@ +""" +tests for models +""" + +from django.utils import unittest +from django.test.client import RequestFactory + +from djeuscan.views import index + + +class ViewsTests(unittest.TestCase): + pass diff --git a/euscanwww/djeuscan/urls.py b/euscanwww/djeuscan/urls.py index 74ba23e..3ba8a3a 100644 --- a/euscanwww/djeuscan/urls.py +++ b/euscanwww/djeuscan/urls.py @@ -43,7 +43,7 @@ urlpatterns = patterns('djeuscan.views', # Global stuff (r'^api/', include('djeuscan.api.urls')), - (r'^$', 'index'), + url(r'^$', 'index', name="index"), url(r'^feed/$', GlobalFeed(), name='global_feed'), (r'^about/$', 'about'), (r'^about/api$', 'api'), diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py index c5cc46a..3ce67ec 100644 --- a/euscanwww/djeuscan/views.py +++ b/euscanwww/djeuscan/views.py @@ -1,42 +1,37 @@ +""" Views """ + from annoying.decorators import render_to from django.http import Http404 from django.shortcuts import get_object_or_404 -from django.db.models import Sum, Max +from django.db.models import Max -from models import Version, Package, Herd, Maintainer, EuscanResult, VersionLog -from forms import WorldForm, PackagesForm +from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \ + VersionLog +from djeuscan.forms import WorldForm, PackagesForm import charts -""" Views """ - @render_to('euscan/index.html') def index(request): - ctx = {} - ctx['n_packaged'] = charts.xint( - Package.objects.aggregate(Sum('n_packaged'))['n_packaged__sum'] - ) - ctx['n_overlay'] = charts.xint( - Package.objects.aggregate(Sum('n_overlay'))['n_overlay__sum'] - ) - ctx['n_versions'] = charts.xint( - Package.objects.aggregate(Sum('n_versions'))['n_versions__sum'] - ) - ctx['n_upstream'] = ctx['n_versions'] - ctx['n_packaged'] - \ - ctx['n_overlay'] - ctx['n_packages'] = Package.objects.count() - ctx['n_herds'] = Herd.objects.count() - ctx['n_maintainers'] = Maintainer.objects.count() - + context = { + 'n_packaged': Package.objects.n_packaged(), + 'n_overlay': Package.objects.n_overlay(), + 'n_versions': Package.objects.n_versions(), + 'n_upstream': Package.objects.n_upstream(), + 'n_packages': Package.objects.count(), + 'n_herds': Herd.objects.count(), + 'n_maintainers': Maintainer.objects.count(), + } try: - ctx['last_scan'] = EuscanResult.objects.get( + context['last_scan'] = \ + EuscanResult.objects.get( id=EuscanResult.objects.aggregate(Max('id'))['id__max'] ).datetime except EuscanResult.DoesNotExist: - ctx['last_scan'] = None + context['last_scan'] = None - return ctx + return context @render_to('euscan/logs.html') @@ -46,13 +41,7 @@ def logs(request): @render_to('euscan/categories.html') def categories(request): - categories = Package.objects.values('category').annotate( - n_packaged=Sum('n_packaged'), - n_overlay=Sum('n_overlay'), - n_versions=Sum('n_versions') - ) - - return {'categories': categories} + return {'categories': Package.objects.categories()} @render_to('euscan/category.html') @@ -61,7 +50,7 @@ def category(request, category): packages = packages.select_related( 'last_version_gentoo', 'last_version_overlay', 'last_version_upstream' ) - print dir(packages[0]) + if not packages: raise Http404 return {'category': category, 'packages': packages} @@ -69,13 +58,7 @@ def category(request, category): @render_to('euscan/herds.html') def herds(request): - # FIXME: optimize the query, it uses 'LEFT OUTER JOIN' instead of - # 'INNER JOIN' - herds = Package.objects.filter(herds__isnull=False) - herds = herds.values('herds__herd').annotate( - n_packaged=Sum('n_packaged'), - n_overlay=Sum('n_overlay'), - n_versions=Sum('n_versions')) + herds = Package.objects.herds() return {'herds': herds} @@ -91,16 +74,7 @@ def herd(request, herd): @render_to('euscan/maintainers.html') def maintainers(request): - maintainers = Package.objects.filter(maintainers__isnull=False) - maintainers = maintainers.values( - 'maintainers__id', 'maintainers__name', 'maintainers__email' - ) - maintainers = maintainers.annotate( - n_packaged=Sum('n_packaged'), - n_overlay=Sum('n_overlay'), - n_versions=Sum('n_versions') - ) - + maintainers = Package.objects.maintainers() return {'maintainers': maintainers} @@ -116,9 +90,7 @@ def maintainer(request, maintainer_id): @render_to('euscan/overlays.html') def overlays(request): - overlays = Package.objects.values('version__overlay') - overlays = overlays.exclude(version__overlay='') - overlays = overlays.distinct() + overlays = Package.objects.overlays() return {'overlays': overlays} diff --git a/euscanwww/euscanwww/models.py b/euscanwww/euscanwww/models.py new file mode 100644 index 0000000..e69de29 diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py index 8512442..34b7766 100644 --- a/euscanwww/euscanwww/settings.py +++ b/euscanwww/euscanwww/settings.py @@ -132,15 +132,15 @@ MIDDLEWARE_CLASSES = ( # 'django.middleware.clickjacking.XFrameOptionsMiddleware', ) -CACHE_MIDDLEWARE_SECONDS=3600 -CACHE_MIDDLEWARE_ANONYMOUS_ONLY=True +CACHE_MIDDLEWARE_SECONDS = 3600 +CACHE_MIDDLEWARE_ANONYMOUS_ONLY = True ROOT_URLCONF = 'euscanwww.urls' # Python dotted path to the WSGI application used by Django's runserver. WSGI_APPLICATION = 'euscanwww.wsgi.application' -FORCE_SCRIPT_NAME="" +FORCE_SCRIPT_NAME = "" TEMPLATE_DIRS = ( os.path.join(SITE_ROOT, 'templates'), @@ -166,6 +166,7 @@ INSTALLED_APPS = ( # Uncomment the next line to enable admin documentation: # 'django.contrib.admindocs', 'south', + 'euscanwww', 'djeuscan', ) diff --git a/euscanwww/euscanwww/tests.py b/euscanwww/euscanwww/tests.py new file mode 100644 index 0000000..597e5b0 --- /dev/null +++ b/euscanwww/euscanwww/tests.py @@ -0,0 +1,45 @@ +""" +System tests for euscanwww +""" + +from urllib import urlencode + +from django.utils import unittest +from django.test.client import Client +from django.core.urlresolvers import reverse + + +class SystemTestCase(unittest.TestCase): + """ + Base class for system tests + """ + + def setUp(self): + self.client = Client() + + def get(self, url_name, *args, **kwargs): + param = kwargs.pop("param", None) + if param: + url = "%s?%s" % (reverse(url_name, args=args, kwargs=kwargs), + urlencode(param)) + else: + url = reverse(url_name, args=args, kwargs=kwargs) + return self.client.get(url) + + def post(self, url_name, *args, **kwargs): + data = kwargs.pop("data", {}) + url = reverse(url_name, args=args, kwargs=kwargs) + return self.client.post(url, data) + + +class NavigationTest(SystemTestCase): + """ + Test main pages + """ + + def test_index(self): + """ + Test index + """ + response = self.get("index") + self.assertEqual(response.status_code, 200)