diff --git a/euscanwww/djeuscan/migrations/0005_auto__add_field_version_version_type__add_field_version_handler__add_f.py b/euscanwww/djeuscan/migrations/0005_auto__add_field_version_version_type__add_field_version_handler__add_f.py new file mode 100644 index 0000000..64eda3c --- /dev/null +++ b/euscanwww/djeuscan/migrations/0005_auto__add_field_version_version_type__add_field_version_handler__add_f.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'Version.version_type' + db.add_column('djeuscan_version', 'version_type', + self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True), + keep_default=False) + + # Adding field 'Version.handler' + db.add_column('djeuscan_version', 'handler', + self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True), + keep_default=False) + + # Adding field 'Version.confidence' + db.add_column('djeuscan_version', 'confidence', + self.gf('django.db.models.fields.IntegerField')(default=0), + keep_default=False) + + def backwards(self, orm): + # Deleting field 'Version.version_type' + db.delete_column('djeuscan_version', 'version_type') + + # Deleting field 'Version.handler' + db.delete_column('djeuscan_version', 'handler') + + # Deleting field 'Version.confidence' + db.delete_column('djeuscan_version', 'confidence') + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'djeuscan.categoryassociation': { + 'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.categorylog': { + 'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.euscanresult': { + 'Meta': {'object_name': 'EuscanResult'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'djeuscan.herd': { + 'Meta': {'object_name': 'Herd'}, + 'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'djeuscan.herdassociation': { + 'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.herdlog': { + 'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.log': { + 'Meta': {'object_name': 'Log'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'djeuscan.maintainer': { + 'Meta': {'object_name': 'Maintainer'}, + 'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.maintainerassociation': { + 'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.maintainerlog': { + 'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}) + }, + 'djeuscan.package': { + 'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}), + 'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}), + 'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.packageassociation': { + 'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.refreshpackagequery': { + 'Meta': {'object_name': 'RefreshPackageQuery'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'}) + }, + 'djeuscan.version': { + 'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'}, + 'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'version_type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.versionlog': { + 'Meta': {'object_name': 'VersionLog'}, + 'action': ('django.db.models.fields.IntegerField', [], {}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.worldlog': { + 'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + } + } + + complete_apps = ['djeuscan'] \ No newline at end of file diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py index 936b2fc..5bca337 100644 --- a/euscanwww/djeuscan/models.py +++ b/euscanwww/djeuscan/models.py @@ -126,6 +126,10 @@ class Version(models.Model): urls = models.TextField(blank=True) alive = models.BooleanField(default=True, db_index=True) + version_type = models.CharField(max_length=128, blank=True) + handler = models.CharField(max_length=128, blank=True) + confidence = models.IntegerField(default=0) + class Meta: unique_together = ['package', 'slot', 'revision', 'version', 'overlay'] diff --git a/euscanwww/djeuscan/processing/scan_portage.py b/euscanwww/djeuscan/processing/scan_portage.py index fdebc57..34456ea 100644 --- a/euscanwww/djeuscan/processing/scan_portage.py +++ b/euscanwww/djeuscan/processing/scan_portage.py @@ -2,10 +2,13 @@ import subprocess import portage import os import re +from xml.dom.minidom import parseString from django.db.transaction import commit_on_success from django.core.management.color import color_style +from euscan.helpers import get_version_type + from djeuscan.processing import FakeLogger from djeuscan.models import Package, Version, VersionLog @@ -83,11 +86,7 @@ class ScanPortage(object): @commit_on_success def scan(self, query=None): - env = os.environ - env['MY'] = "/-: []\n" - - cmd = ['eix', '--format', '', '--pure-packages', - '-x'] + cmd = ['eix', '--xml', '--pure-packages', '-x'] if query: cmd.extend(['--exact', query]) @@ -96,7 +95,7 @@ class ScanPortage(object): Version.objects.filter(packaged=True).update(alive=False) self.logger.info('done') - output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\ + output = subprocess.Popen(cmd, stdout=subprocess.PIPE).\ communicate()[0] output = output.strip().strip('\n') @@ -118,50 +117,42 @@ class ScanPortage(object): ) return - output = output.split('\n') - packages = {} + dom = parseString(output) - line_re = re.compile( - r'^(?P.*?):(?P.*?) \[(?P.*?)\]$' - ) + for category_tag in dom.getElementsByTagName("category"): + for package_tag in category_tag.getElementsByTagName("package"): + cat = category_tag.getAttribute("name") + pkg = package_tag.getAttribute("name") + homepage_tags = package_tag.getElementsByTagName("homepage") + homepage = homepage_tags[0].firstChild.nodeValue \ + if homepage_tags else "" + desc_tags = package_tag.getElementsByTagName("description") + desc = desc_tags[0].firstChild.nodeValue if desc_tags else "" - package = None + package = self.store_package(cat, pkg, homepage, desc) - for line in output: - match = line_re.match(line) - - if not match: - continue - - cpv = match.group('cpv') - slot = match.group('slot') - overlay = match.group('overlay') - - cat, pkg, ver, rev = portage.catpkgsplit(cpv) - - packages['%s/%s' % (cat, pkg)] = True - - if not package or not \ - (cat == package.category and pkg == package.name): - package = self.store_package(cat, pkg) - - self.store_version(package, cpv, slot, overlay) + for version_tag in package_tag.getElementsByTagName("version"): + cpv = "%s/%s-%s" % (cat, pkg, + version_tag.getAttribute("id")) + slot = version_tag.getAttribute("slot") + overlay = version_tag.getAttribute("overlay") + self.store_version(package, cpv, slot, overlay) if self.purge_packages and not query: for package in Package.objects.all(): - cp = "%s/%s" % (package.category, package.name) - if cp not in packages: - self.logger.info('- [p] %s' % (package)) - package.delete() + self.logger.info('- [p] %s' % (package)) + package.delete() - def store_package(self, cat, pkg): + def store_package(self, cat, pkg, homepage, description): created = False obj = self.cache_get_package(cat, pkg) if not obj: obj, created = Package.objects.get_or_create( category=cat, - name=pkg + name=pkg, + homepage=homepage, + description=description, ) self.cache_store_package(obj) @@ -197,7 +188,13 @@ class ScanPortage(object): package=package, slot=slot, revision=rev, version=ver, overlay=overlay, - defaults={"alive": True, "packaged": True} + defaults={ + "alive": True, + "packaged": True, + "version_type": get_version_type(ver), + "confidence": 100, + "handler": "portage" + } ) if not created: # Created objects have defaults values obj.alive = True diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py index bb34836..5629b7b 100644 --- a/pym/euscan/helpers.py +++ b/pym/euscan/helpers.py @@ -44,7 +44,7 @@ def get_version_type(version): if token in gentoo_types: types.append(token) if types: - return types[0] + return types[0] # TODO: consider returning all types return "release"