euscanwww: try to keep trace of versions change

Signed-off-by: Corentin Chary <corentincj@iksaif.net>
2011-08-25 15:39:54 +02:00
parent 7570453bc2
commit d0fa19bc1d
15 changed files with 585 additions and 79 deletions
--- a/euscanwww/euscan/management/commands/scan-metadata.py
+++ b/euscanwww/euscan/management/commands/scan-metadata.py
@@ -14,6 +14,8 @@ from euscanwww.euscan.models import Package, Herd, Maintainer
 from gentoolkit.query import Query
 from gentoolkit.errors import GentoolkitFatalError

+from progressbar import ProgressBar, Bar, ETA, Percentage
+
 class Command(BaseCommand):
    _overlays = {}

@@ -28,6 +30,11 @@ class Command(BaseCommand):
            dest='quiet',
            default=False,
            help='Be quiet'),
+        make_option('--progress',
+            action='store_true',
+            dest='progress',
+            default=False,
+            help='Display progress'),
        )
    args = '<package package ...>'
    help = 'Scans metadata and fills database'
@@ -36,24 +43,41 @@ class Command(BaseCommand):
        if len(args) == 0 and options['all'] == False:
            raise CommandError('You must specify a package or use --all')

-        if not options['quiet']:
-            self.stdout.write('Scanning metadata...\n')
+        if not self.stdout.isatty():
+            options['progress'] = False
+
+        if options['progress']:
+            widgets = ['Scanning metadata: ', Percentage(), ' ', Bar(), ' ', ETA()]
+            if len(args):
+                count = len(args)
+            else:
+                count = Package.objects.count()
+            pbar = ProgressBar(widgets=widgets, maxval=count).start()
+            i = 0
+        else:
+            pbar = None

        if len(args) == 0:
            for pkg in Package.objects.all():
                self.scan(options, '%s/%s' % (pkg.category, pkg.name))
+                if pbar:
+                    pbar.update(i)
+                    i += 1
        else:
            for package in args:
                self.scan(options, package)
+                if pbar:
+                    pbar.update(i)
+                    i += 1

-        if not options['quiet']:
-            self.stdout.write('Done.\n')
+        if pbar:
+            pbar.finish()

    @commit_on_success
    def scan(self, options, query=None):
        matches = Query(query).find(
                include_masked=True,
-                in_installed=False
+                in_installed=False,
        )

        if not matches:
@@ -73,6 +97,8 @@ class Command(BaseCommand):
        except GentoolkitFatalError, err:
            sys.stderr.write(self.style.ERROR("Gentoolkit fatal error: '%s'\n" % str(err)))

+        if created and not options['quiet']:
+            sys.stdout.write('+ [p] %s/%s\n' % (pkg.category, pkg.name))
        if pkg.metadata:
            obj.herds.clear()
            obj.maintainers.clear()
@@ -85,9 +111,6 @@ class Command(BaseCommand):
                maintainer = self.store_maintainer(options, maintainer.name, maintainer.email)
                obj.maintainers.add(maintainer)

-        if not options['quiet']:
-            sys.stdout.write('[p] %s/%s\n' % (pkg.category, pkg.name))
-
        obj.save()

    def store_herd(self, options, name, email):
@@ -99,7 +122,7 @@ class Command(BaseCommand):

        if created or herd.email != email:
            if not options['quiet']:
-                sys.stdout.write('[h] %s <%s>\n' % (name, email))
+                sys.stdout.write('+ [h] %s <%s>\n' % (name, email))

            herd.email = email
            herd.save()
@@ -116,7 +139,7 @@ class Command(BaseCommand):

        if created:
            if not options['quiet']:
-                sys.stdout.write('[m] %s <%s>\n' % (name.encode('utf-8'), email))
+                sys.stdout.write('+ [m] %s <%s>\n' % (name.encode('utf-8'), email))

            maintainer.name = name
            maintainer.save()
--- a/euscanwww/euscan/management/commands/scan-portage.py
+++ b/euscanwww/euscan/management/commands/scan-portage.py
@@ -9,7 +9,7 @@ from optparse import make_option

 from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand, CommandError
-from euscanwww.euscan.models import Package, Version
+from euscanwww.euscan.models import Package, Version, VersionLog

 class Command(BaseCommand):
    _overlays = {}
@@ -20,11 +20,16 @@ class Command(BaseCommand):
            dest='all',
            default=False,
            help='Scan all packages'),
-        make_option('--purge',
+        make_option('--purge-packages',
            action='store_true',
-            dest='purge',
+            dest='purge-packages',
            default=False,
            help='Purge old packages'),
+        make_option('--purge-versions',
+            action='store_true',
+            dest='purge-versions',
+            default=False,
+            help='Purge old versions'),
        make_option('--quiet',
            action='store_true',
            dest='quiet',
@@ -47,6 +52,9 @@ class Command(BaseCommand):
            for package in args:
                self.scan(options, package)

+        if options['purge-versions']:
+            self.purge_versions(options)
+
        if not options['quiet']:
            self.stdout.write('Done.\n')

@@ -90,9 +98,9 @@ class Command(BaseCommand):
        if len(output) == 0:
            if not query:
                return
-            if options['purge']:
+            if options['purge-packages']:
                if not options['quiet']:
-                    sys.stdout.write('[gc] %s\n' % (query))
+                    sys.stdout.write('- [p] %s\n' % (query))
                if '/' in query:
                    cat, pkg = portage.catsplit(query)
                    Package.objects.filter(category=cat, name=pkg).delete()
@@ -128,12 +136,12 @@ class Command(BaseCommand):

            self.store_version(options, package, cpv, slot, overlay)

-        if options['purge'] and not query:
+        if options['purge-packages'] and not query:
            for package in Package.objects.all():
                cp = "%s/%s" % (package.category, package.name)
                if cp not in packages:
                    if not options['quiet']:
-                        sys.stdout.write('[gc] %s\n' % (cp))
+                        sys.stdout.write('- [p] %s\n' % (package))
                    package.delete()

    def store_package(self, options, cat, pkg):
@@ -141,10 +149,10 @@ class Command(BaseCommand):

        if created:
            if not options['quiet']:
-                sys.stdout.write('[p] %s/%s\n' % (cat, pkg))
+                sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))

-        # Delete previous versions to handle incremental scan correctly
-        Version.objects.filter(package=obj, packaged=True).delete()
+        ' Set all versions dead, then set found versions alive and delete old versions '
+        Version.objects.filter(package=obj, packaged=True).update(alive=False)

        obj.n_packaged = 0
        obj.n_overlay = 0
@@ -163,23 +171,53 @@ class Command(BaseCommand):
        else:
            overlay = 'gentoo'

-        if not options['quiet']:
-            sys.stdout.write('[v] %s:%s [%s]\n' % (cpv, slot, overlay))
-
        obj, created = Version.objects.get_or_create(package=package, slot=slot,
                                                     revision=rev, version=ver,
                                                     overlay=overlay)
-
-        if created or not package.n_packaged:
-            if overlay == 'gentoo':
-                package.n_packaged += 1
-            else:
-                package.n_overlay += 1
-        if created:
-            package.n_versions += 1
-
-        package.save()
-
+        obj.alive = True
        obj.packaged = True
        obj.save()

+        ''' nothing to do (note: it can't be an upstream version because overlay can't be empty here) '''
+        if not created:
+            return
+
+        if not options['quiet']:
+            sys.stdout.write('+ [v] %s \n' % (obj))
+
+        if overlay == 'gentoo':
+            package.n_packaged += 1
+        else:
+            package.n_overlay += 1
+        package.n_versions += 1
+        package.save()
+
+        entry = VersionLog.objects.create(package=obj.package, action=VersionLog.VERSION_ADDED)
+        entry.slot = obj.slot
+        entry.revision = obj.revision
+        entry.version = obj.version
+        entry.overlay = obj.overlay
+        entry.save()
+
+    @commit_on_success
+    def purge_versions(self, options):
+        ' For each dead versions '
+        for version in Version.objects.filter(packaged=True, alive=False):
+            entry = VersionLog.objects.create(package=version.package, action=VersionLog.VERSION_REMOVED)
+            entry.slot = version.slot
+            entry.revision = version.revision
+            entry.version = version.version
+            entry.overlay = version.overlay
+            entry.save()
+
+            if version.overlay == 'gentoo':
+                version.package.n_packaged -= 1
+            else:
+                version.package.n_overlay -= 1
+            version.package.n_versions -= 1
+            version.package.save()
+
+            if not options['quiet']:
+                sys.stdout.write('- [v] %s\n' % (version))
+        Version.objects.filter(packaged=False, alive=False).delete()
+
--- a/euscanwww/euscan/management/commands/scan-upstream.py
+++ b/euscanwww/euscan/management/commands/scan-upstream.py
@@ -11,7 +11,7 @@ from optparse import make_option

 from django.db.transaction import commit_on_success
 from django.core.management.base import BaseCommand, CommandError
-from euscanwww.euscan.models import Package, Version, EuscanResult
+from euscanwww.euscan.models import Package, Version, EuscanResult, VersionLog

 class Command(BaseCommand):
    _overlays = {}
@@ -27,6 +27,11 @@ class Command(BaseCommand):
            dest='feed',
            default=False,
            help='Read euscan output from stdin'),
+        make_option('--purge-versions',
+            action='store_true',
+            dest='purge-versions',
+            default=False,
+            help='Purge old versions'),
        make_option('--quiet',
            action='store_true',
            dest='quiet',
@@ -42,6 +47,8 @@ class Command(BaseCommand):

        if options['feed']:
            self.parse_output(options, sys.stdin)
+            if options['purge-versions']:
+                self.purge_versions(options)
            return

        if not options['quiet']:
@@ -57,10 +64,12 @@ class Command(BaseCommand):

        self.scan(options, packages)

+        if options['purge-versions']:
+            self.purge_versions(options)
+
        if not options['quiet']:
            self.stdout.write('Done.\n')

-    @commit_on_success
    def scan(self, options, packages=None):
        for package in packages:
            cmd = ['euscan', package]
@@ -70,6 +79,7 @@ class Command(BaseCommand):

            self.parse_output(options, output)

+    @commit_on_success
    def parse_output(self, options, output):
        from portage.versions import _cp

@@ -120,12 +130,11 @@ class Command(BaseCommand):

        obj, created = Package.objects.get_or_create(category=cat, name=pkg)

-        if created:
-            if not options['quiet']:
-                sys.stdout.write('[p] %s/%s\n' % (cat, pkg))
+        if created and not options['quiet']:
+            sys.stdout.write('+ [p] %s/%s\n' % (cat, pkg))

-        # Delete previous versions to handle incremental scan correctly
-        Version.objects.filter(package=obj, packaged=False).delete()
+        ' Set all versions dead, then set found versions alive and delete old versions '
+        Version.objects.filter(package=obj, packaged=False).update(alive=False)

        obj.n_versions = Version.objects.filter(package=obj).count()
        obj.save()
@@ -137,15 +146,44 @@ class Command(BaseCommand):
                                                     revision='r0', version=ver,
                                                     overlay='')

-        if created or not obj.packaged:
+        obj.alive = True
+        obj.urls = url
+        obj.packaged = False
+        obj.save()
+
+        ''' If it's not a new version, just update the object and continue '''
+        if not created:
+            return
+
+        if not options['quiet']:
+            sys.stdout.write('+ [u] %s %s\n' % (obj, url))
+
+        entry = VersionLog.objects.create(package=package, action=VersionLog.VERSION_ADDED)
+        entry.slot = ''
+        entry.revision = 'r0'
+        entry.version = ver
+        entry.overlay = ''
+        entry.save()
+
+        package.n_versions += 1
+        package.save()
+
+
+    @commit_on_success
+    def purge_versions(self, options):
+        ' For each dead versions '
+        for version in Version.objects.filter(packaged=False, alive=False):
+            entry = VersionLog.objects.create(package=version.package, action=VersionLog.VERSION_REMOVED)
+            entry.slot = version.slot
+            entry.revision = version.revision
+            entry.version = version.version
+            entry.overlay = version.overlay
+            entry.save()
+
+            version.package.n_versions -= 1
+            version.package.save()
+
            if not options['quiet']:
-                sys.stdout.write('[u] %s/%s-%s %s\n' % (package.category, package.name,
-                                                        ver, url))
+                sys.stdout.write('- [u] %s %s\n' % (version, version.urls))
+        Version.objects.filter(packaged=False, alive=False).delete()

-            obj.urls = url
-            obj.packaged = False
-            obj.save()
-
-        if created:
-            package.n_versions += 1
-            package.save()
--- a/euscanwww/euscan/management/commands/update-counters.py
+++ b/euscanwww/euscan/management/commands/update-counters.py
@@ -29,8 +29,10 @@ class Command(BaseCommand):
        herds = {}
        maintainers = {}

-        # Could be done using raw SQL queries, but I don't have time for that
-        # right now ...
+        '''
+        Could be done using raw SQL queries, but I don't have time for that
+        right now ...
+        '''

        wlog = WorldLog()
        wlog.datetime = now
@@ -100,19 +102,19 @@ class Command(BaseCommand):

        for clog in categories.values():
            if not options['quiet']:
-                self.stdout.write('[c] %s\n' % clog)
+                self.stdout.write('+ [cl] %s\n' % clog)
            charts.rrd_update('category-%s' % clog.category, now, clog)
            clog.save()

        for hlog in herds.values():
            if not options['quiet']:
-                self.stdout.write('[h] %s\n' % hlog)
+                self.stdout.write('+ [hl] %s\n' % hlog)
            charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
            hlog.save()

        for mlog in maintainers.values():
            if not options['quiet']:
-                self.stdout.write('[m] %s\n' % mlog)
+                self.stdout.write('+ [ml] %s\n' % mlog)
            charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
            mlog.save()