euscanwww: scan_metadata populates db from herds.xml

* Every herd has a m2m relation with maintainers
* New method populate_herds_and_maintainers to fill data in from herds.xml

Signed-off-by: volpino <fox91@anche.no>

euscanwww/djeuscan/migrations/0013_auto.py

@@ -0,0 +1,205 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding M2M table for field maintainers on 'Herd'
+        db.create_table('djeuscan_herd_maintainers', (
+            ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
+            ('herd', models.ForeignKey(orm['djeuscan.herd'], null=False)),
+            ('maintainer', models.ForeignKey(orm['djeuscan.maintainer'], null=False))
+        ))
+        db.create_unique('djeuscan_herd_maintainers', ['herd_id', 'maintainer_id'])
+
+    def backwards(self, orm):
+        # Removing M2M table for field maintainers on 'Herd'
+        db.delete_table('djeuscan_herd_maintainers')
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'djeuscan.categoryassociation': {
+            'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.categorylog': {
+            'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.euscanresult': {
+            'Meta': {'object_name': 'EuscanResult'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'djeuscan.herd': {
+            'Meta': {'object_name': 'Herd'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
+            'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'})
+        },
+        'djeuscan.herdassociation': {
+            'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.herdlog': {
+            'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
+            'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        },
+        'djeuscan.log': {
+            'Meta': {'object_name': 'Log'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'djeuscan.maintainer': {
+            'Meta': {'object_name': 'Maintainer'},
+            'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.maintainerassociation': {
+            'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.maintainerlog': {
+            'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
+            'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
+        },
+        'djeuscan.overlayassociation': {
+            'Meta': {'unique_together': "(['user', 'overlay'],)", 'object_name': 'OverlayAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.package': {
+            'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
+            'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'djeuscan.packageassociation': {
+            'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'djeuscan.problemreport': {
+            'Meta': {'object_name': 'ProblemReport'},
+            'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'message': ('django.db.models.fields.TextField', [], {}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'subject': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Version']", 'null': 'True', 'blank': 'True'})
+        },
+        'djeuscan.refreshpackagequery': {
+            'Meta': {'object_name': 'RefreshPackageQuery'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'djeuscan.version': {
+            'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
+            'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
+            'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'ebuild_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
+            'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'metadata_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
+        },
+        'djeuscan.versionlog': {
+            'Meta': {'object_name': 'VersionLog'},
+            'action': ('django.db.models.fields.IntegerField', [], {}),
+            'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
+            'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
+            'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
+        },
+        'djeuscan.worldlog': {
+            'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
+            'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
+        }
+    }
+
+    complete_apps = ['djeuscan']

euscanwww/djeuscan/models.py

@@ -23,6 +23,7 @@ class Herd(models.Model):
                             validators=[validate_name])
     email = models.CharField(max_length=128, blank=True, null=True,
                              validators=[validate_email])
+    maintainers = models.ManyToManyField("Maintainer")
 
     def __unicode__(self):
         if self.email:

euscanwww/djeuscan/processing/scan/scan_metadata.py

@@ -1,4 +1,9 @@
+import os
+
 from gentoolkit.query import Query
+from gentoolkit.dbapi import PORTDB
+
+import xml.etree.cElementTree as etree
 
 from django.db.transaction import commit_on_success
 from django.core.management.color import color_style
@@ -87,7 +92,7 @@ class ScanMetadata(object):
                 self.logger.error(
                     self.style.ERROR("Bad maintainer: '%s' '%s'" % \
                                          (maintainer.name, maintainer.email))
-                    )
+                )
         obj.save()
 
     def store_herd(self, name, email):
@@ -125,9 +130,41 @@ class ScanMetadata(object):
             )
         return maintainer
 
+    def populate_herds_and_maintainers(self):
+        self.logger.info("Populating herds and maintainers from herds.xml...")
+
+        herds_xml_path = os.path.join(
+            PORTDB.settings["PORTDIR"], "metadata", "herds.xml"
+        )
+        try:
+            self._herdstree = etree.parse(herds_xml_path)
+        except IOError:
+            return None
+
+        for herd_node in self._herdstree.getiterator('herd'):
+            herd_name = herd_node.findtext('name')
+            herd_email = herd_node.findtext('email')
+
+            try:
+                herd = self.store_herd(herd_name, herd_email)
+            except ValidationError:  # just skip invalid data
+                continue
+
+            herd.maintainers.clear()  # clear previous data
+
+            for maintainer_node in herd_node:
+                if maintainer_node.tag == "maintainer":
+                    maintainer_name = maintainer_node.findtext('name')
+                    maintainer_email = maintainer_node.findtext('email')
+
+                    maintainer = self.store_maintainer(
+                        maintainer_name, maintainer_email
+                    )
+                    herd.maintainers.add(maintainer)
+
 
 @commit_on_success
-def scan_metadata(packages=None, category=None, logger=None):
+def scan_metadata(packages=None, category=None, logger=None, populate=True):
     scan_handler = ScanMetadata(logger=logger)
 
     if category:
@@ -135,6 +172,9 @@ def scan_metadata(packages=None, category=None, logger=None):
     elif not packages:
         packages = Package.objects.all()
 
+    if populate:
+        scan_handler.populate_herds_and_maintainers()
+
     for pkg in packages:
         if isinstance(pkg, Package):
             scan_handler.scan('%s/%s' % (pkg.category, pkg.name), pkg)