euscanwww: Tweaked processing.scan_portage to use eix --xml

Signed-off-by: volpino <fox91@anche.no>

euscanwww/djeuscan/migrations/0005_auto__add_field_version_version_type__add_field_version_handler__add_f.py

@@ -0,0 +1,197 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'Version.version_type'
+        db.add_column('djeuscan_version', 'version_type',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True),
+                      keep_default=False)
+
+        # Adding field 'Version.handler'
+        db.add_column('djeuscan_version', 'handler',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True),
+                      keep_default=False)
+
+        # Adding field 'Version.confidence'
+        db.add_column('djeuscan_version', 'confidence',
+                      self.gf('django.db.models.fields.IntegerField')(default=0),
+                      keep_default=False)
+
+    def backwards(self, orm):
+        # Deleting field 'Version.version_type'
+        db.delete_column('djeuscan_version', 'version_type')
+
+        # Deleting field 'Version.handler'
+        db.delete_column('djeuscan_version', 'handler')
+
+        # Deleting field 'Version.confidence'
+        db.delete_column('djeuscan_version', 'confidence')
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'djeuscan.categoryassociation': {
+            'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.categorylog': {
+            'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.euscanresult': {
+            'Meta': {'object_name': 'EuscanResult'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'djeuscan.herd': {
+            'Meta': {'object_name': 'Herd'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
+            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'djeuscan.herdassociation': {
+            'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.herdlog': {
+            'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.log': {
+            'Meta': {'object_name': 'Log'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'djeuscan.maintainer': {
+            'Meta': {'object_name': 'Maintainer'},
+            'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.maintainerassociation': {
+            'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.maintainerlog': {
+            'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
+        },
+        'djeuscan.package': {
+            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
+            'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.packageassociation': {
+            'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.refreshpackagequery': {
+            'Meta': {'object_name': 'RefreshPackageQuery'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
+        },
+        'djeuscan.version': {
+            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
+            'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'version_type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
+        },
+        'djeuscan.versionlog': {
+            'Meta': {'object_name': 'VersionLog'},
+            'action': ('django.db.models.fields.IntegerField', [], {}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.worldlog': {
+            'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        }
+    }
+
+    complete_apps = ['djeuscan']

euscanwww/djeuscan/models.py

@@ -126,6 +126,10 @@ class Version(models.Model):
     urls = models.TextField(blank=True)
     alive = models.BooleanField(default=True, db_index=True)
 
+    version_type = models.CharField(max_length=128, blank=True)
+    handler = models.CharField(max_length=128, blank=True)
+    confidence = models.IntegerField(default=0)
+
     class Meta:
         unique_together = ['package', 'slot', 'revision', 'version', 'overlay']
 

euscanwww/djeuscan/processing/scan_portage.py

@@ -2,10 +2,13 @@ import subprocess
 import portage
 import os
 import re
+from xml.dom.minidom import parseString
 
 from django.db.transaction import commit_on_success
 from django.core.management.color import color_style
 
+from euscan.helpers import get_version_type
+
 from djeuscan.processing import FakeLogger
 from djeuscan.models import Package, Version, VersionLog
 
@@ -83,11 +86,7 @@ class ScanPortage(object):
 
     @commit_on_success
     def scan(self, query=None):
-        env = os.environ
+        cmd = ['eix', '--xml', '--pure-packages', '-x']
-        env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
-
-        cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages',
-               '-x']
         if query:
             cmd.extend(['--exact', query])
 
@@ -96,7 +95,7 @@ class ScanPortage(object):
             Version.objects.filter(packaged=True).update(alive=False)
             self.logger.info('done')
 
-        output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
+        output = subprocess.Popen(cmd, stdout=subprocess.PIPE).\
             communicate()[0]
         output = output.strip().strip('\n')
 
@@ -118,50 +117,42 @@ class ScanPortage(object):
                 )
             return
 
-        output = output.split('\n')
+        dom = parseString(output)
-        packages = {}
 
-        line_re = re.compile(
+        for category_tag in dom.getElementsByTagName("category"):
-            r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$'
+            for package_tag in category_tag.getElementsByTagName("package"):
-        )
+                cat = category_tag.getAttribute("name")
+                pkg = package_tag.getAttribute("name")
+                homepage_tags = package_tag.getElementsByTagName("homepage")
+                homepage = homepage_tags[0].firstChild.nodeValue \
+                           if homepage_tags else ""
+                desc_tags = package_tag.getElementsByTagName("description")
+                desc = desc_tags[0].firstChild.nodeValue if desc_tags else ""
 
-        package = None
+                package = self.store_package(cat, pkg, homepage, desc)
 
-        for line in output:
+                for version_tag in package_tag.getElementsByTagName("version"):
-            match = line_re.match(line)
+                    cpv = "%s/%s-%s" % (cat, pkg,
-
+                                        version_tag.getAttribute("id"))
-            if not match:
+                    slot = version_tag.getAttribute("slot")
-                continue
+                    overlay = version_tag.getAttribute("overlay")
-
+                    self.store_version(package, cpv, slot, overlay)
-            cpv = match.group('cpv')
-            slot = match.group('slot')
-            overlay = match.group('overlay')
-
-            cat, pkg, ver, rev = portage.catpkgsplit(cpv)
-
-            packages['%s/%s' % (cat, pkg)] = True
-
-            if not package or not \
-               (cat == package.category and pkg == package.name):
-                package = self.store_package(cat, pkg)
-
-            self.store_version(package, cpv, slot, overlay)
 
         if self.purge_packages and not query:
             for package in Package.objects.all():
-                cp = "%s/%s" % (package.category, package.name)
+                self.logger.info('- [p] %s' % (package))
-                if cp not in packages:
+                package.delete()
-                    self.logger.info('- [p] %s' % (package))
-                    package.delete()
 
-    def store_package(self, cat, pkg):
+    def store_package(self, cat, pkg, homepage, description):
         created = False
         obj = self.cache_get_package(cat, pkg)
 
         if not obj:
             obj, created = Package.objects.get_or_create(
                 category=cat,
-                name=pkg
+                name=pkg,
+                homepage=homepage,
+                description=description,
             )
             self.cache_store_package(obj)
 
@@ -197,7 +188,13 @@ class ScanPortage(object):
                 package=package, slot=slot,
                 revision=rev, version=ver,
                 overlay=overlay,
-                defaults={"alive": True, "packaged": True}
+                defaults={
+                    "alive": True,
+                    "packaged": True,
+                    "version_type": get_version_type(ver),
+                    "confidence": 100,
+                    "handler": "portage"
+                }
             )
         if not created:  # Created objects have defaults values
             obj.alive = True

pym/euscan/helpers.py

@@ -44,7 +44,7 @@ def get_version_type(version):
         if token in gentoo_types:
             types.append(token)
     if types:
-        return types[0]
+        return types[0]  # TODO: consider returning all types
     return "release"