Merge remote branch 'fox/master'

euscanwww/djeuscan/management/commands/list-packages.py

@@ -1,56 +0,0 @@
-from optparse import make_option
-
-from django.core.management.base import BaseCommand
-from djeuscan.models import Package
-
-
-class Command(BaseCommand):
-    _overlays = {}
-    help = 'List packages'
-
-    option_list = BaseCommand.option_list + (
-        make_option('--after',
-            action='store',
-            dest='after',
-            default=False,
-            help='After package'),
-        make_option('--before',
-            action='store',
-            dest='before',
-            default=False,
-            help='Before package'),
-        make_option('--limit',
-            action='store',
-            dest='limit',
-            default=False,
-            help='limit'),
-        )
-
-    def handle(self, *args, **options):
-        after = None
-        before = None
-
-        if options['after']:
-            category, name = options['after'].split('/')
-            after = Package.objects.get(category=category, name=name)
-
-        if options['before']:
-            category, name = options['before'].split('/')
-            before = Package.objects.get(category=category, name=name)
-
-        packages = Package.objects
-
-        if after or before:
-            if after:
-                packages = packages.filter(id__gte=after.id)
-            if before:
-                packages = packages.filter(id__lte=before.id)
-        else:
-            packages = packages.all()
-
-        if options['limit']:
-            packages = packages[:int(options['limit'])]
-
-        for pkg in packages:
-            self.stdout.write('%s/%s\n' % (pkg.category, pkg.name))
-        self.stdout.close()

euscanwww/djeuscan/management/commands/list_packages.py

@@ -0,0 +1,64 @@
+import sys
+from optparse import make_option
+
+from django.core.management.base import BaseCommand
+from djeuscan.models import Package
+
+
+def list_packages(stdout=None, **options):
+    if stdout is None:
+        stdout = sys.stdout
+
+    after = None
+    before = None
+
+    if options['after']:
+        category, name = options['after'].split('/')
+        after = Package.objects.get(category=category, name=name)
+
+    if options['before']:
+        category, name = options['before'].split('/')
+        before = Package.objects.get(category=category, name=name)
+
+    packages = Package.objects
+
+    if after or before:
+        if after:
+            packages = packages.filter(id__gte=after.id)
+        if before:
+            packages = packages.filter(id__lte=before.id)
+    else:
+        packages = packages.all()
+
+    if options['limit']:
+        packages = packages[:int(options['limit'])]
+
+    for pkg in packages:
+        stdout.write('%s/%s\n' % (pkg.category, pkg.name))
+    stdout.close()
+
+
+class Command(BaseCommand):
+    _overlays = {}
+    help = 'List packages'
+
+    option_list = BaseCommand.option_list + (
+        make_option('--after',
+            action='store',
+            dest='after',
+            default=False,
+            help='After package'),
+        make_option('--before',
+            action='store',
+            dest='before',
+            default=False,
+            help='Before package'),
+        make_option('--limit',
+            action='store',
+            dest='limit',
+            default=False,
+            help='limit'),
+        )
+
+    def handle(self, *args, **options):
+        list_packages(self.stdout, **options)

euscanwww/djeuscan/management/commands/regen-rrds.py

@@ -1,23 +0,0 @@
-from django.core.management.base import BaseCommand
-from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
-from djeuscan import charts
-
-
-class Command(BaseCommand):
-    _overlays = {}
-    help = 'Regenerate rrd database'
-
-    def handle(self, *args, **options):
-        for wlog in WorldLog.objects.all():
-            charts.rrd_update('world', wlog.datetime, wlog)
-
-        for clog in CategoryLog.objects.all():
-            charts.rrd_update('category-%s' % clog.category,
-                              clog.datetime, clog)
-
-        for hlog in HerdLog.objects.all():
-            charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
-
-        for mlog in MaintainerLog.objects.all():
-            charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
-                              mlog.datetime, mlog)

euscanwww/djeuscan/management/commands/regen_rrds.py

@@ -0,0 +1,30 @@
+from django.core.management.base import BaseCommand
+from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
+from djeuscan import charts
+
+
+def regen_rrds():
+    """
+    Regenerates the rrd database
+    """
+    for wlog in WorldLog.objects.all():
+        charts.rrd_update('world', wlog.datetime, wlog)
+
+    for clog in CategoryLog.objects.all():
+        charts.rrd_update('category-%s' % clog.category,
+                          clog.datetime, clog)
+
+    for hlog in HerdLog.objects.all():
+        charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
+
+    for mlog in MaintainerLog.objects.all():
+        charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
+                          mlog.datetime, mlog)
+
+
+class Command(BaseCommand):
+    _overlays = {}
+    help = 'Regenerate rrd database'
+
+    def handle(self, *args, **options):
+        regen_rrds()

euscanwww/djeuscan/management/commands/scan_metadata.py

@@ -2,47 +2,23 @@ import sys
 
 from optparse import make_option
 
-from django.db.transaction import commit_on_success
-from django.core.management.base import BaseCommand
-from djeuscan.models import Package, Herd, Maintainer
-
 from gentoolkit.query import Query
 from gentoolkit.errors import GentoolkitFatalError
 
+from django.db.transaction import commit_on_success
+from django.core.management.base import BaseCommand
+from django.core.management.color import color_style
 
-class Command(BaseCommand):
+from djeuscan.models import Package, Herd, Maintainer
-    _overlays = {}
 
-    option_list = BaseCommand.option_list + (
+
-        make_option('--all',
+class ScanMetadata(object):
-            action='store_true',
+    def __init__(self, quiet):
-            dest='all',
+        self.quiet = quiet
-            default=False,
+        self.style = color_style()
-            help='Scan all packages'),
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
-        )
-    args = '<package package ...>'
-    help = 'Scans metadata and fills database'
 
     @commit_on_success
-    def handle(self, *args, **options):
+    def run(self, query=None, obj=None):
-        self.options = options
-
-        if options['all']:
-            for pkg in Package.objects.all():
-                self.scan('%s/%s' % (pkg.category, pkg.name), pkg)
-        elif len(args):
-            for package in args:
-                self.scan(package)
-        else:
-            for package in sys.stdin.readlines():
-                self.scan(package[:-1])
-
-    def scan(self, query=None, obj=None):
         matches = Query(query).find(
             include_masked=True,
             in_installed=False,
@@ -76,7 +52,7 @@ class Command(BaseCommand):
                 )
             )
 
-        if created and not self.options['quiet']:
+        if created and not self.quiet:
             sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
 
         if pkg.metadata:
@@ -125,9 +101,12 @@ class Command(BaseCommand):
             name = '{nil}'
         name = name.strip("\r").strip("\n").strip("\t").strip()
 
-        herd, created = Herd.objects.get_or_create(herd=name)
+        herd, created = Herd.objects.get_or_create(
+            herd=name,
+            defaults={"email": email}
+        )
 
-        if created and not self.options['quiet']:
+        if created and not self.quiet:
             sys.stdout.write('+ [h] %s <%s>\n' % (name, email))
 
         herd.email = email
@@ -141,17 +120,48 @@ class Command(BaseCommand):
         if not name:
             name = '{nil}'
 
-        maintainer, created = Maintainer.objects.get_or_create(email=email)
+        maintainer, created = Maintainer.objects.get_or_create(
+            email=email,
+            defaults={"name": name}
+        )
 
         if created:
-            if not self.options['quiet']:
+            if not self.quiet:
                 sys.stdout.write(
                     '+ [m] %s <%s>\n' % (name.encode('utf-8'), email)
                 )
-
-            if not maintainer.name or \
-               name not in [maintainer.name, email, '{nil}']:
-                maintainer.name = name
-            maintainer.save()
-
         return maintainer
+
+
+class Command(BaseCommand):
+    _overlays = {}
+
+    option_list = BaseCommand.option_list + (
+        make_option('--all',
+            action='store_true',
+            dest='all',
+            default=False,
+            help='Scan all packages'),
+        make_option('--quiet',
+            action='store_true',
+            dest='quiet',
+            default=False,
+            help='Be quiet'),
+        )
+    args = '<package package ...>'
+    help = 'Scans metadata and fills database'
+
+    def handle(self, *args, **options):
+        self.options = options
+
+        scan_metadata = ScanMetadata(quiet=options["quiet"])
+
+        if options['all']:
+            for pkg in Package.objects.all():
+                scan_metadata.run('%s/%s' % (pkg.category, pkg.name), pkg)
+        elif len(args) > 0:
+            for package in args:
+                scan_metadata.run(package)
+        else:
+            for package in sys.stdin.readlines():
+                scan_metadata.run(package[:-1])

euscanwww/djeuscan/management/commands/scan_portage.py

@@ -8,49 +8,22 @@ from optparse import make_option
 
 from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand
+from django.core.management.color import color_style
+
 from djeuscan.models import Package, Version, VersionLog
 
 
-class Command(BaseCommand):
+class ScanPortage(object):
-    _overlays = {}
+    def __init__(self, stdout=None, **options):
+        if stdout is None:
+            self.stdout = sys.stdout
+        else:
+            self.stdout = stdout
 
-    option_list = BaseCommand.option_list + (
+        self.options = options
-        make_option('--all',
+        self.style = color_style()
-            action='store_true',
+        self._cache = {'packages': {}, 'versions': {}}
-            dest='all',
+        self._overlays = None
-            default=False,
-            help='Scan all packages'),
-        make_option('--purge-packages',
-            action='store_true',
-            dest='purge-packages',
-            default=False,
-            help='Purge old packages'),
-        make_option('--purge-versions',
-            action='store_true',
-            dest='purge-versions',
-            default=False,
-            help='Purge old versions'),
-        make_option('--no-log',
-            action='store_true',
-            dest='no-log',
-            default=False,
-            help='Don\'t store logs'),
-        make_option('--prefetch',
-            action='store_true',
-            dest='prefetch',
-            default=False,
-            help=('Prefetch all versions and packages from DB to '
-                  'speedup full scan process.')),
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
-        )
-    args = '[package package ...]'
-    help = 'Scans portage tree and fills database'
-
-    _cache = {'packages': {}, 'versions': {}}
 
     def cache_hash_package(self, category, name):
         return '%s/%s' % (category, name)
@@ -84,38 +57,6 @@ class Command(BaseCommand):
         )
         self._cache['versions'][key] = version
 
-    def handle(self, *args, **options):
-        self.options = options
-
-        if not options['quiet']:
-            self.stdout.write('Scanning portage tree...\n')
-
-        if options['prefetch']:
-            if not options['quiet']:
-                self.stdout.write('Prefetching objects...')
-                self.stdout.flush()
-            for package in Package.objects.all():
-                self.cache_store_package(package)
-            for version in Version.objects.select_related('package').all():
-                self.cache_store_version(version)
-            if not options['quiet']:
-                self.stdout.write('done\n')
-
-        if options['all']:
-            self.scan()
-        elif len(args):
-            for package in args:
-                self.scan(package)
-        else:
-            for package in sys.stdin.readlines():
-                self.scan(package[:-1])
-
-        if options['purge-versions']:
-            self.purge_versions(options)
-
-        if not options['quiet']:
-            self.stdout.write('Done.\n')
-
     def overlays(self):
         if self._overlays:
             return self._overlays
@@ -143,7 +84,7 @@ class Command(BaseCommand):
         return self._overlays
 
     @commit_on_success
-    def scan(self, query=None):
+    def run(self, query=None):
         env = os.environ
         env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
 
@@ -263,9 +204,10 @@ class Command(BaseCommand):
             obj, created = Version.objects.get_or_create(
                 package=package, slot=slot,
                 revision=rev, version=ver,
-                overlay=overlay
+                overlay=overlay,
+                defaults={"alive": True, "packaged": True}
             )
-
+            if not created:
                 obj.alive = True
                 obj.packaged = True
                 obj.save()
@@ -291,17 +233,19 @@ class Command(BaseCommand):
         if self.options['no-log']:
             return
 
-        entry = VersionLog.objects.create(package=obj.package,
+        VersionLog.objects.create(
-                                          action=VersionLog.VERSION_ADDED)
+            package=obj.package,
-        entry.slot = obj.slot
+            action=VersionLog.VERSION_ADDED,
-        entry.revision = obj.revision
+            slot=obj.slot,
-        entry.version = obj.version
+            revision=obj.revision,
-        entry.overlay = obj.overlay
+            version=obj.version,
-        entry.save()
+            overlay=obj.overlay
+        )
+
 
 @commit_on_success
-    def purge_versions(self, options):
+def purge_versions(options):
-        ' For each dead versions '
+    # For each dead versions
     for version in Version.objects.filter(packaged=True, alive=False):
         if version.overlay == 'gentoo':
             version.package.n_packaged -= 1
@@ -310,20 +254,91 @@ class Command(BaseCommand):
         version.package.n_versions -= 1
         version.package.save()
 
-            if not self.options['quiet']:
+        if not options['quiet']:
             sys.stdout.write('- [v] %s\n' % (version))
 
-            if self.options['no-log']:
+        if options['no-log']:
             continue
 
-            entry = VersionLog.objects.create(
+        VersionLog.objects.create(
             package=version.package,
-                action=VersionLog.VERSION_REMOVED
+            action=VersionLog.VERSION_REMOVED,
+            slot=version.slot,
+            revision=version.revision,
+            version=version.version,
+            overlay=version.overlay
         )
-            entry.slot = version.slot
-            entry.revision = version.revision
-            entry.version = version.version
-            entry.overlay = version.overlay
-            entry.save()
 
     Version.objects.filter(packaged=True, alive=False).delete()
+
+
+class Command(BaseCommand):
+    _overlays = {}
+
+    option_list = BaseCommand.option_list + (
+        make_option('--all',
+            action='store_true',
+            dest='all',
+            default=False,
+            help='Scan all packages'),
+        make_option('--purge-packages',
+            action='store_true',
+            dest='purge-packages',
+            default=False,
+            help='Purge old packages'),
+        make_option('--purge-versions',
+            action='store_true',
+            dest='purge-versions',
+            default=False,
+            help='Purge old versions'),
+        make_option('--no-log',
+            action='store_true',
+            dest='no-log',
+            default=False,
+            help='Don\'t store logs'),
+        make_option('--prefetch',
+            action='store_true',
+            dest='prefetch',
+            default=False,
+            help=('Prefetch all versions and packages from DB to '
+                  'speedup full scan process.')),
+        make_option('--quiet',
+            action='store_true',
+            dest='quiet',
+            default=False,
+            help='Be quiet'),
+        )
+    args = '[package package ...]'
+    help = 'Scans portage tree and fills database'
+
+    def handle(self, *args, **options):
+        scan_portage = ScanPortage(stdout=self.stdout, **options)
+
+        if not options['quiet']:
+            self.stdout.write('Scanning portage tree...\n')
+
+        if options['prefetch']:
+            if not options['quiet']:
+                self.stdout.write('Prefetching objects...')
+                self.stdout.flush()
+            for package in Package.objects.all():
+                scan_portage.cache_store_package(package)
+            for version in Version.objects.select_related('package').all():
+                scan_portage.cache_store_version(version)
+            if not options['quiet']:
+                self.stdout.write('done\n')
+
+        if options['all']:
+            scan_portage.run()
+        elif len(args):
+            for package in args:
+                scan_portage.run(package)
+        else:
+            for package in sys.stdin.readlines():
+                scan_portage.run(package[:-1])
+
+        if options['purge-versions']:
+            purge_versions(options)
+
+        if not options['quiet']:
+            self.stdout.write('Done.\n')

euscanwww/djeuscan/management/commands/scan_upstream.py

@@ -2,16 +2,149 @@ import subprocess
 import portage
 import sys
 import re
-
 from StringIO import StringIO
 from optparse import make_option
+from collections import defaultdict
 
 from django.utils import timezone
 from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand
+
 from djeuscan.models import Package, Version, EuscanResult, VersionLog
 
 
+class ScanUpstream(object):
+    def __init__(self, options=None):
+        if options is None:
+            self.options = defaultdict(None)
+        else:
+            self.options = options
+
+    def run(self, packages=None):
+        for package in packages:
+            cmd = ['euscan', package]
+
+            fp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE)
+            output = StringIO(fp.communicate()[0])
+
+            self.parse_output(output)
+
+    def store_result(self, package, log):
+        # Remove previous logs
+        EuscanResult.objects.filter(package=package).delete()
+
+        obj = EuscanResult()
+        obj.package = package
+        obj.result = log
+        obj.datetime = timezone.now()
+        obj.save()
+
+    def store_package(self, cpv):
+        cat, pkg, ver, rev = portage.catpkgsplit(cpv)
+
+        obj, created = Package.objects.get_or_create(category=cat, name=pkg)
+
+        if created and not self.options['quiet']:
+            sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
+
+        # Set all versions dead, then set found versions alive and
+        # delete old versions
+        Version.objects.filter(package=obj, packaged=False).update(alive=False)
+
+        return obj
+
+    def store_version(self, package, ver, url):
+        obj, created = Version.objects.get_or_create(
+            package=package, slot='', revision='r0', version=ver, overlay='',
+            defaults={"alive": True, "urls": url, "packaged": True}
+        )
+        if not created:
+            obj.alive = True
+            obj.urls = url
+            obj.packaged = False
+            obj.save()
+
+        # If it's not a new version, just update the object and continue
+        if not created:
+            return
+
+        if not self.options['quiet']:
+            sys.stdout.write('+ [u] %s %s\n' % (obj, url))
+
+        VersionLog.objects.create(
+            package=package,
+            action=VersionLog.VERSION_ADDED,
+            slot='',
+            revision='r0',
+            version=ver,
+            overlay=''
+        )
+
+        package.n_versions += 1
+        package.save()
+
+    @commit_on_success
+    def parse_output(self, output):
+        from portage.versions import _cp
+
+        package_re = re.compile(
+            r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$'
+        )
+        version_re = re.compile(
+            r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$'
+        )
+
+        package = None
+        log = ""
+
+        while True:
+            line = output.readline()
+            if line == '':
+                break
+            match = package_re.match(line)
+            if match:
+                if package:
+                    self.store_result(package, log)
+
+                cpv = match.group('cpv')
+                package = self.store_package(cpv)
+                log = line
+                continue
+
+            log += line
+
+            match = version_re.match(line)
+            if match:
+                ver = match.group('ver')
+                url = match.group('url')
+                self.store_version(package, ver, url)
+
+        if package:
+            self.store_result(package, log)
+
+
+@commit_on_success
+def purge_versions(options):
+    # For each dead versions
+    for version in Version.objects.filter(packaged=False, alive=False):
+        VersionLog.objects.create(
+            package=version.package,
+            action=VersionLog.VERSION_REMOVED,
+            slot=version.slot,
+            revision=version.revision,
+            version=version.version,
+            overlay=version.overlay
+        )
+
+        version.package.n_versions -= 1
+        version.package.save()
+
+        if not options['quiet']:
+            sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
+    Version.objects.filter(packaged=False, alive=False).delete()
+
+
 class Command(BaseCommand):
     _overlays = {}
 
@@ -41,10 +174,12 @@ class Command(BaseCommand):
     help = 'Scans metadata and fills database'
 
     def handle(self, *args, **options):
+        scan_upstream = ScanUpstream(options)
+
         if options['feed']:
-            self.parse_output(options, sys.stdin)
+            scan_upstream.parse_output(options, sys.stdin)
             if options['purge-versions']:
-                self.purge_versions(options)
+                purge_versions(options)
             return
 
         if not options['quiet']:
@@ -60,133 +195,10 @@ class Command(BaseCommand):
         else:
             packages = [package[:-1] for package in sys.stdin.readlines()]
 
-        self.scan(options, packages)
+        scan_upstream.run(packages)
 
         if options['purge-versions']:
-            self.purge_versions(options)
+            purge_versions(options)
 
         if not options['quiet']:
             self.stdout.write('Done.\n')
-
-    def scan(self, options, packages=None):
-        for package in packages:
-            cmd = ['euscan', package]
-
-            fp = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-            output = StringIO(fp.communicate()[0])
-
-            self.parse_output(options, output)
-
-    @commit_on_success
-    def parse_output(self, options, output):
-        from portage.versions import _cp
-
-        package_re = re.compile(
-            r'^ \* (?P<cpv>' + _cp + ') \[(?P<overlay>.*?)\]$'
-        )
-        version_re = re.compile(
-            r'^Upstream Version: (?P<ver>.*?) (?P<url>.*?)$'
-        )
-
-        package = None
-        log = ""
-
-        while True:
-            line = output.readline()
-            if line == '':
-                break
-            match = package_re.match(line)
-            if match:
-                if package:
-                    self.store_result(options, package, log)
-
-                cpv = match.group('cpv')
-                package = self.store_package(options, cpv)
-                log = line
-                continue
-
-            log += line
-
-            match = version_re.match(line)
-            if match:
-                ver = match.group('ver')
-                url = match.group('url')
-                self.store_version(options, package, ver, url)
-
-        if package:
-            self.store_result(options, package, log)
-
-    def store_result(self, options, package, log):
-        # Remove previous logs
-        EuscanResult.objects.filter(package=package).delete()
-
-        obj = EuscanResult()
-        obj.package = package
-        obj.result = log
-        obj.datetime = timezone.now()
-        obj.save()
-
-    def store_package(self, options, cpv):
-        cat, pkg, ver, rev = portage.catpkgsplit(cpv)
-
-        obj, created = Package.objects.get_or_create(category=cat, name=pkg)
-
-        if created and not options['quiet']:
-            sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))
-
-        # Set all versions dead, then set found versions alive and
-        # delete old versions
-        Version.objects.filter(package=obj, packaged=False).update(alive=False)
-
-        return obj
-
-    def store_version(self, options, package, ver, url):
-        obj, created = Version.objects.get_or_create(
-            package=package, slot='', revision='r0', version=ver, overlay=''
-        )
-
-        obj.alive = True
-        obj.urls = url
-        obj.packaged = False
-        obj.save()
-
-        ''' If it's not a new version, just update the object and continue '''
-        if not created:
-            return
-
-        if not options['quiet']:
-            sys.stdout.write('+ [u] %s %s\n' % (obj, url))
-
-        entry = VersionLog.objects.create(
-            package=package,
-            action=VersionLog.VERSION_ADDED
-        )
-        entry.slot = ''
-        entry.revision = 'r0'
-        entry.version = ver
-        entry.overlay = ''
-        entry.save()
-
-        package.n_versions += 1
-        package.save()
-
-    @commit_on_success
-    def purge_versions(self, options):
-        ' For each dead versions '
-        for version in Version.objects.filter(packaged=False, alive=False):
-            entry = VersionLog.objects.create(
-                package=version.package,
-                action=VersionLog.VERSION_REMOVED
-            )
-            entry.slot = version.slot
-            entry.revision = version.revision
-            entry.version = version.version
-            entry.overlay = version.overlay
-            entry.save()
-
-            version.package.n_versions -= 1
-            version.package.save()
-
-            if not options['quiet']:
-                sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
-        Version.objects.filter(packaged=False, alive=False).delete()

euscanwww/djeuscan/management/commands/update-counters.py

@@ -1,199 +0,0 @@
-from optparse import make_option
-
-from django.db.transaction import commit_on_success
-from django.core.management.base import BaseCommand
-from django.utils import timezone
-
-from djeuscan.models import Package, Herd, Maintainer, Version
-from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
-from djeuscan import charts
-
-from distutils.version import StrictVersion, LooseVersion
-
-
-def compare_versions(version1, version2):
-    try:
-        return cmp(StrictVersion(version1), StrictVersion(version2))
-    # in case of abnormal version number, fall back to LooseVersion
-    except ValueError:
-        return cmp(LooseVersion(version1), LooseVersion(version2))
-
-
-class Command(BaseCommand):
-    _overlays = {}
-    help = 'Update counters'
-
-    option_list = BaseCommand.option_list + (
-        make_option('--quiet',
-            action='store_true',
-            dest='quiet',
-            default=False,
-            help='Be quiet'),
-        make_option('--fast',
-            action='store_true',
-            dest='fast',
-            default=False,
-            help='Skip sanity checks'),
-        make_option('--nolog',
-            action='store_true',
-            dest='nolog',
-            default=False,
-            help='Skip logs'),
-
-        )
-
-    @commit_on_success
-    def handle(self, *args, **options):
-        now = timezone.now()
-
-        categories = {}
-        herds = {}
-        maintainers = {}
-
-        wlog = None
-
-        if not options['nolog']:
-            wlog = WorldLog()
-            wlog.datetime = now
-
-            for cat in Package.objects.values('category').distinct():
-                clog = CategoryLog()
-                clog.datetime = now
-                clog.category = cat['category']
-                categories[clog.category] = clog
-
-            for herd in Herd.objects.all():
-                hlog = HerdLog()
-                hlog.datetime = now
-                hlog.herd = herd
-                herds[herd.id] = hlog
-
-            for maintainer in Maintainer.objects.all():
-                mlog = MaintainerLog()
-                mlog.datetime = now
-                mlog.maintainer = maintainer
-                maintainers[maintainer.id] = mlog
-
-        package_queryset = Package.objects.all()
-
-        n_versions = {}
-        n_packaged = {}
-        n_overlay = {}
-
-        last_versions_gentoo = {}
-        last_versions_overlay = {}
-        last_versions_upstream = {}
-
-        def add_safe(storage, key):
-            if key not in storage:
-                storage[key] = 1
-            else:
-                storage[key] += 1
-
-        def add_last_ver(storage, version):
-            key = version['package_id']
-            if key not in storage:
-                storage[key] = version
-                return
-            if version['version'].startswith('9999'):
-                return
-            if compare_versions(storage[key]['version'],
-                                version['version']) < 0:
-                storage[key] = version
-
-        if not options['fast']:
-            attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
-            for version in Version.objects.all().values(*attrs):
-                overlay, packaged = version['overlay'], version['packaged']
-                package_id = version['package_id']
-
-                add_safe(n_versions, package_id)
-
-                if not packaged:
-                    add_last_ver(last_versions_upstream, version)
-                    continue
-                if overlay == 'gentoo':
-                    add_safe(n_packaged, package_id)
-                    add_last_ver(last_versions_gentoo, version)
-                else:
-                    add_safe(n_overlay, package_id)
-                    add_last_ver(last_versions_overlay, version)
-
-        for package in package_queryset.select_related('herds', 'maintainers'):
-            if not options['fast']:
-                package.n_versions = n_versions.get(package.id, 0)
-                package.n_packaged = n_packaged.get(package.id, 0)
-                package.n_overlay = n_overlay.get(package.id, 0)
-
-                default = {'id': None}
-                package.last_version_gentoo_id = last_versions_gentoo.get(
-                    package.id, default
-                )['id']
-                package.last_version_overlay_id = last_versions_overlay.get(
-                    package.id, default
-                )['id']
-                package.last_version_upstream_id = last_versions_upstream.get(
-                    package.id, default
-                )['id']
-
-                package.save()
-
-            n_packages_gentoo = int(package.n_packaged == package.n_versions)
-            n_packages_overlay = int(package.n_overlay and package.n_packaged \
-                                     + package.n_overlay == package.n_versions)
-            n_packages_outdated = int(package.n_packaged + package.n_overlay \
-                                      < package.n_versions)
-
-            def update_row(storage, key):
-                storage[key].n_packages_gentoo += n_packages_gentoo
-                storage[key].n_packages_overlay += n_packages_overlay
-                storage[key].n_packages_outdated += n_packages_outdated
-
-                storage[key].n_versions_gentoo += package.n_packaged
-                storage[key].n_versions_overlay += package.n_overlay
-                storage[key].n_versions_upstream += package.n_versions - \
-                                                    package.n_packaged - \
-                                                    package.n_overlay
-
-            def update_log(storage, qs):
-                for row in qs:
-                    update_row(storage, row['id'])
-
-            if not options['nolog']:
-                update_log(herds, package.herds.all().values('id'))
-                update_log(maintainers, package.maintainers.all().values('id'))
-                update_row(categories, package.category)
-
-                wlog.n_packages_gentoo += n_packages_gentoo
-                wlog.n_packages_overlay += n_packages_overlay
-                wlog.n_packages_outdated += n_packages_outdated
-
-                wlog.n_versions_gentoo += package.n_packaged
-                wlog.n_versions_overlay += package.n_overlay
-                wlog.n_versions_upstream += package.n_versions - \
-                                            package.n_packaged - \
-                                            package.n_overlay
-
-        if options['nolog']:
-            return
-
-        for clog in categories.values():
-            if not options['quiet']:
-                self.stdout.write('+ [cl] %s\n' % clog)
-            charts.rrd_update('category-%s' % clog.category, now, clog)
-            clog.save()
-
-        for hlog in herds.values():
-            if not options['quiet']:
-                self.stdout.write('+ [hl] %s\n' % hlog)
-            charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
-            hlog.save()
-
-        for mlog in maintainers.values():
-            if not options['quiet']:
-                self.stdout.write('+ [ml] %s\n' % mlog)
-            charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
-            mlog.save()
-
-        charts.rrd_update('world', now, wlog)
-        wlog.save()

euscanwww/djeuscan/management/commands/update_counters.py

@@ -0,0 +1,208 @@
+import sys
+from optparse import make_option
+
+from django.db.transaction import commit_on_success
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from djeuscan.models import Package, Herd, Maintainer, Version
+from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
+from djeuscan import charts
+
+from distutils.version import StrictVersion, LooseVersion
+
+
+def compare_versions(version1, version2):
+    try:
+        return cmp(StrictVersion(version1), StrictVersion(version2))
+    # in case of abnormal version number, fall back to LooseVersion
+    except ValueError:
+        return cmp(LooseVersion(version1), LooseVersion(version2))
+
+
+def add_safe(storage, key):
+    if key not in storage:
+        storage[key] = 1
+    else:
+        storage[key] += 1
+
+
+def add_last_ver(storage, version):
+    key = version['package_id']
+    if key not in storage:
+        storage[key] = version
+        return
+    if version['version'].startswith('9999'):
+        return
+    if compare_versions(storage[key]['version'],
+                        version['version']) < 0:
+        storage[key] = version
+
+
+@commit_on_success
+def update_counters(stdout=None, **options):
+    if stdout is None:
+        stdout = sys.stdout
+
+    now = timezone.now()
+
+    categories = {}
+    herds = {}
+    maintainers = {}
+
+    wlog = None
+
+    if not options['nolog']:
+        wlog = WorldLog()
+        wlog.datetime = now
+
+        for cat in Package.objects.values('category').distinct():
+            clog = CategoryLog()
+            clog.datetime = now
+            clog.category = cat['category']
+            categories[clog.category] = clog
+
+        for herd in Herd.objects.all():
+            hlog = HerdLog()
+            hlog.datetime = now
+            hlog.herd = herd
+            herds[herd.id] = hlog
+
+        for maintainer in Maintainer.objects.all():
+            mlog = MaintainerLog()
+            mlog.datetime = now
+            mlog.maintainer = maintainer
+            maintainers[maintainer.id] = mlog
+
+    package_queryset = Package.objects.all()
+
+    n_versions = {}
+    n_packaged = {}
+    n_overlay = {}
+
+    last_versions_gentoo = {}
+    last_versions_overlay = {}
+    last_versions_upstream = {}
+
+    if not options['fast']:
+        attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
+        for version in Version.objects.all().values(*attrs):
+            overlay, packaged = version['overlay'], version['packaged']
+            package_id = version['package_id']
+
+            add_safe(n_versions, package_id)
+
+            if not packaged:
+                add_last_ver(last_versions_upstream, version)
+                continue
+            if overlay == 'gentoo':
+                add_safe(n_packaged, package_id)
+                add_last_ver(last_versions_gentoo, version)
+            else:
+                add_safe(n_overlay, package_id)
+                add_last_ver(last_versions_overlay, version)
+
+    for package in package_queryset.select_related('herds', 'maintainers'):
+        if not options['fast']:
+            package.n_versions = n_versions.get(package.id, 0)
+            package.n_packaged = n_packaged.get(package.id, 0)
+            package.n_overlay = n_overlay.get(package.id, 0)
+
+            default = {'id': None}
+            package.last_version_gentoo_id = last_versions_gentoo.get(
+                package.id, default
+            )['id']
+            package.last_version_overlay_id = last_versions_overlay.get(
+                package.id, default
+            )['id']
+            package.last_version_upstream_id = last_versions_upstream.get(
+                package.id, default
+            )['id']
+
+            package.save()
+
+        n_packages_gentoo = int(package.n_packaged == package.n_versions)
+        n_packages_overlay = int(package.n_overlay and package.n_packaged \
+                                 + package.n_overlay == package.n_versions)
+        n_packages_outdated = int(package.n_packaged + package.n_overlay \
+                                  < package.n_versions)
+
+        def update_row(storage, key):
+            storage[key].n_packages_gentoo += n_packages_gentoo
+            storage[key].n_packages_overlay += n_packages_overlay
+            storage[key].n_packages_outdated += n_packages_outdated
+
+            storage[key].n_versions_gentoo += package.n_packaged
+            storage[key].n_versions_overlay += package.n_overlay
+            storage[key].n_versions_upstream += package.n_versions - \
+                                                package.n_packaged - \
+                                                package.n_overlay
+
+        def update_log(storage, qs):
+            for row in qs:
+                update_row(storage, row['id'])
+
+        if not options['nolog']:
+            update_log(herds, package.herds.all().values('id'))
+            update_log(maintainers, package.maintainers.all().values('id'))
+            update_row(categories, package.category)
+
+            wlog.n_packages_gentoo += n_packages_gentoo
+            wlog.n_packages_overlay += n_packages_overlay
+            wlog.n_packages_outdated += n_packages_outdated
+
+            wlog.n_versions_gentoo += package.n_packaged
+            wlog.n_versions_overlay += package.n_overlay
+            wlog.n_versions_upstream += package.n_versions - \
+                                        package.n_packaged - \
+                                        package.n_overlay
+
+    if options['nolog']:
+        return
+
+    for clog in categories.values():
+        if not options['quiet']:
+            stdout.write('+ [cl] %s\n' % clog)
+        charts.rrd_update('category-%s' % clog.category, now, clog)
+        clog.save()
+
+    for hlog in herds.values():
+        if not options['quiet']:
+            stdout.write('+ [hl] %s\n' % hlog)
+        charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
+        hlog.save()
+
+    for mlog in maintainers.values():
+        if not options['quiet']:
+            stdout.write('+ [ml] %s\n' % mlog)
+        charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
+        mlog.save()
+
+    charts.rrd_update('world', now, wlog)
+    wlog.save()
+
+
+class Command(BaseCommand):
+    _overlays = {}
+    help = 'Update counters'
+
+    option_list = BaseCommand.option_list + (
+        make_option('--quiet',
+            action='store_true',
+            dest='quiet',
+            default=False,
+            help='Be quiet'),
+        make_option('--fast',
+            action='store_true',
+            dest='fast',
+            default=False,
+            help='Skip sanity checks'),
+        make_option('--nolog',
+            action='store_true',
+            dest='nolog',
+            default=False,
+            help='Skip logs'),
+    )
+
+    def handle(self, *args, **options):
+        update_counters(stdout=self.stdout, **options)

euscanwww/djeuscan/models.py

@@ -1,11 +1,12 @@
 from django.db import models
 from django.core.validators import RegexValidator, validate_email, URLValidator
+from django.core.exceptions import ValidationError
 
 from djeuscan.managers import PackageManager, VersionLogManager, \
     EuscanResultManager
 
 
-validate_category = RegexValidator("^\w+?-\w+?$")
+validate_category = RegexValidator("^(?:\w+?-\w+?)|virtual$")
 validate_name = RegexValidator("^\S+?$")
 validate_revision = RegexValidator("^r\d+?$")
 validate_url = URLValidator()
@@ -56,7 +57,7 @@ class Package(models.Model):
     category = models.CharField(max_length=128, validators=[validate_category])
     name = models.CharField(max_length=128, validators=[validate_name])
     description = models.TextField(blank=True)
-    homepage = models.TextField(blank=True, validators=[validate_url])
+    homepage = models.TextField(blank=True)
     herds = models.ManyToManyField(Herd, blank=True)
     maintainers = models.ManyToManyField(Maintainer, blank=True)
 
@@ -89,6 +90,18 @@ class Package(models.Model):
 
     def save(self, *args, **kwargs):
         self.full_clean()
+
+        # Clean urls, accept only real urls
+        urls = []
+        for url in self.homepages:
+            try:
+                validate_url(url)
+            except ValidationError:
+                pass
+            else:
+                urls.append(url)
+        self.homepage = " ".join(urls)
+
         super(Package, self).save(*args, **kwargs)
 
     @property

setup.py

@@ -87,8 +87,9 @@ setup(
         'https://github.com/iksaif/euscan/tarball/' +
         ('master' if __version__ == '9999' else ('euscan-%s' % __version__))
     ),
-    install_requires=['Django==1.4', 'django-annoying', 'South',
+    install_requires=['Django==1.4', 'django-annoying==0..7.6', 'South==0.7.4',
-                      'django-piston', 'BeautifulSoup', "matplotlib"],
+                      'django-piston==0.2.3', 'BeautifulSoup==3.2.1',
+                      'matplotlib==1.1.0', 'django-celery==2.5.5'],
     package_dir={'': 'pym'},
     packages=packages,
     package_data={},