euscanwww: Tweaked processing.scan_portage to use eix --xml
Signed-off-by: volpino <fox91@anche.no>
This commit is contained in:
volpino
parent
f9c9a4bfd8
commit
d7e0a4ca16
@ -0,0 +1,197 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import datetime
|
||||
from south.db import db
|
||||
from south.v2 import SchemaMigration
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Migration(SchemaMigration):
|
||||
|
||||
def forwards(self, orm):
|
||||
# Adding field 'Version.version_type'
|
||||
db.add_column('djeuscan_version', 'version_type',
|
||||
self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True),
|
||||
keep_default=False)
|
||||
|
||||
# Adding field 'Version.handler'
|
||||
db.add_column('djeuscan_version', 'handler',
|
||||
self.gf('django.db.models.fields.CharField')(default='', max_length=128, blank=True),
|
||||
keep_default=False)
|
||||
|
||||
# Adding field 'Version.confidence'
|
||||
db.add_column('djeuscan_version', 'confidence',
|
||||
self.gf('django.db.models.fields.IntegerField')(default=0),
|
||||
keep_default=False)
|
||||
|
||||
def backwards(self, orm):
|
||||
# Deleting field 'Version.version_type'
|
||||
db.delete_column('djeuscan_version', 'version_type')
|
||||
|
||||
# Deleting field 'Version.handler'
|
||||
db.delete_column('djeuscan_version', 'handler')
|
||||
|
||||
# Deleting field 'Version.confidence'
|
||||
db.delete_column('djeuscan_version', 'confidence')
|
||||
|
||||
models = {
|
||||
'auth.group': {
|
||||
'Meta': {'object_name': 'Group'},
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
|
||||
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
|
||||
},
|
||||
'auth.permission': {
|
||||
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
|
||||
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
|
||||
},
|
||||
'auth.user': {
|
||||
'Meta': {'object_name': 'User'},
|
||||
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
|
||||
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
|
||||
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
|
||||
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
|
||||
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
|
||||
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
|
||||
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
|
||||
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
|
||||
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
|
||||
},
|
||||
'contenttypes.contenttype': {
|
||||
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
|
||||
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
|
||||
},
|
||||
'djeuscan.categoryassociation': {
|
||||
'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'},
|
||||
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
|
||||
},
|
||||
'djeuscan.categorylog': {
|
||||
'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
|
||||
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
|
||||
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
|
||||
},
|
||||
'djeuscan.euscanresult': {
|
||||
'Meta': {'object_name': 'EuscanResult'},
|
||||
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
|
||||
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'})
|
||||
},
|
||||
'djeuscan.herd': {
|
||||
'Meta': {'object_name': 'Herd'},
|
||||
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
|
||||
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
|
||||
},
|
||||
'djeuscan.herdassociation': {
|
||||
'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'},
|
||||
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
|
||||
},
|
||||
'djeuscan.herdlog': {
|
||||
'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
|
||||
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
|
||||
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
|
||||
},
|
||||
'djeuscan.log': {
|
||||
'Meta': {'object_name': 'Log'},
|
||||
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
|
||||
},
|
||||
'djeuscan.maintainer': {
|
||||
'Meta': {'object_name': 'Maintainer'},
|
||||
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
|
||||
},
|
||||
'djeuscan.maintainerassociation': {
|
||||
'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'},
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}),
|
||||
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
|
||||
},
|
||||
'djeuscan.maintainerlog': {
|
||||
'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
|
||||
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
|
||||
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
|
||||
},
|
||||
'djeuscan.package': {
|
||||
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
|
||||
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
|
||||
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
|
||||
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
|
||||
'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
|
||||
'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
|
||||
'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
|
||||
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
|
||||
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
|
||||
},
|
||||
'djeuscan.packageassociation': {
|
||||
'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'},
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
|
||||
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
|
||||
},
|
||||
'djeuscan.refreshpackagequery': {
|
||||
'Meta': {'object_name': 'RefreshPackageQuery'},
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
|
||||
},
|
||||
'djeuscan.version': {
|
||||
'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
|
||||
'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
|
||||
'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
|
||||
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
|
||||
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
|
||||
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
|
||||
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
|
||||
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
|
||||
'version_type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
|
||||
},
|
||||
'djeuscan.versionlog': {
|
||||
'Meta': {'object_name': 'VersionLog'},
|
||||
'action': ('django.db.models.fields.IntegerField', [], {}),
|
||||
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
|
||||
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
|
||||
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
|
||||
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
|
||||
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
|
||||
},
|
||||
'djeuscan.worldlog': {
|
||||
'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
|
||||
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
|
||||
}
|
||||
}
|
||||
|
||||
complete_apps = ['djeuscan']
|
||||
@ -126,6 +126,10 @@ class Version(models.Model):
|
||||
urls = models.TextField(blank=True)
|
||||
alive = models.BooleanField(default=True, db_index=True)
|
||||
|
||||
version_type = models.CharField(max_length=128, blank=True)
|
||||
handler = models.CharField(max_length=128, blank=True)
|
||||
confidence = models.IntegerField(default=0)
|
||||
|
||||
class Meta:
|
||||
unique_together = ['package', 'slot', 'revision', 'version', 'overlay']
|
||||
|
||||
|
||||
@ -2,10 +2,13 @@ import subprocess
|
||||
import portage
|
||||
import os
|
||||
import re
|
||||
from xml.dom.minidom import parseString
|
||||
|
||||
from django.db.transaction import commit_on_success
|
||||
from django.core.management.color import color_style
|
||||
|
||||
from euscan.helpers import get_version_type
|
||||
|
||||
from djeuscan.processing import FakeLogger
|
||||
from djeuscan.models import Package, Version, VersionLog
|
||||
|
||||
@ -83,11 +86,7 @@ class ScanPortage(object):
|
||||
|
||||
@commit_on_success
|
||||
def scan(self, query=None):
|
||||
env = os.environ
|
||||
env['MY'] = "<category>/<name>-<version>:<slot> [<overlaynum>]\n"
|
||||
|
||||
cmd = ['eix', '--format', '<availableversions:MY>', '--pure-packages',
|
||||
'-x']
|
||||
cmd = ['eix', '--xml', '--pure-packages', '-x']
|
||||
if query:
|
||||
cmd.extend(['--exact', query])
|
||||
|
||||
@ -96,7 +95,7 @@ class ScanPortage(object):
|
||||
Version.objects.filter(packaged=True).update(alive=False)
|
||||
self.logger.info('done')
|
||||
|
||||
output = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).\
|
||||
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).\
|
||||
communicate()[0]
|
||||
output = output.strip().strip('\n')
|
||||
|
||||
@ -118,50 +117,42 @@ class ScanPortage(object):
|
||||
)
|
||||
return
|
||||
|
||||
output = output.split('\n')
|
||||
packages = {}
|
||||
dom = parseString(output)
|
||||
|
||||
line_re = re.compile(
|
||||
r'^(?P<cpv>.*?):(?P<slot>.*?) \[(?P<overlay>.*?)\]$'
|
||||
)
|
||||
for category_tag in dom.getElementsByTagName("category"):
|
||||
for package_tag in category_tag.getElementsByTagName("package"):
|
||||
cat = category_tag.getAttribute("name")
|
||||
pkg = package_tag.getAttribute("name")
|
||||
homepage_tags = package_tag.getElementsByTagName("homepage")
|
||||
homepage = homepage_tags[0].firstChild.nodeValue \
|
||||
if homepage_tags else ""
|
||||
desc_tags = package_tag.getElementsByTagName("description")
|
||||
desc = desc_tags[0].firstChild.nodeValue if desc_tags else ""
|
||||
|
||||
package = None
|
||||
package = self.store_package(cat, pkg, homepage, desc)
|
||||
|
||||
for line in output:
|
||||
match = line_re.match(line)
|
||||
|
||||
if not match:
|
||||
continue
|
||||
|
||||
cpv = match.group('cpv')
|
||||
slot = match.group('slot')
|
||||
overlay = match.group('overlay')
|
||||
|
||||
cat, pkg, ver, rev = portage.catpkgsplit(cpv)
|
||||
|
||||
packages['%s/%s' % (cat, pkg)] = True
|
||||
|
||||
if not package or not \
|
||||
(cat == package.category and pkg == package.name):
|
||||
package = self.store_package(cat, pkg)
|
||||
|
||||
self.store_version(package, cpv, slot, overlay)
|
||||
for version_tag in package_tag.getElementsByTagName("version"):
|
||||
cpv = "%s/%s-%s" % (cat, pkg,
|
||||
version_tag.getAttribute("id"))
|
||||
slot = version_tag.getAttribute("slot")
|
||||
overlay = version_tag.getAttribute("overlay")
|
||||
self.store_version(package, cpv, slot, overlay)
|
||||
|
||||
if self.purge_packages and not query:
|
||||
for package in Package.objects.all():
|
||||
cp = "%s/%s" % (package.category, package.name)
|
||||
if cp not in packages:
|
||||
self.logger.info('- [p] %s' % (package))
|
||||
package.delete()
|
||||
self.logger.info('- [p] %s' % (package))
|
||||
package.delete()
|
||||
|
||||
def store_package(self, cat, pkg):
|
||||
def store_package(self, cat, pkg, homepage, description):
|
||||
created = False
|
||||
obj = self.cache_get_package(cat, pkg)
|
||||
|
||||
if not obj:
|
||||
obj, created = Package.objects.get_or_create(
|
||||
category=cat,
|
||||
name=pkg
|
||||
name=pkg,
|
||||
homepage=homepage,
|
||||
description=description,
|
||||
)
|
||||
self.cache_store_package(obj)
|
||||
|
||||
@ -197,7 +188,13 @@ class ScanPortage(object):
|
||||
package=package, slot=slot,
|
||||
revision=rev, version=ver,
|
||||
overlay=overlay,
|
||||
defaults={"alive": True, "packaged": True}
|
||||
defaults={
|
||||
"alive": True,
|
||||
"packaged": True,
|
||||
"version_type": get_version_type(ver),
|
||||
"confidence": 100,
|
||||
"handler": "portage"
|
||||
}
|
||||
)
|
||||
if not created: # Created objects have defaults values
|
||||
obj.alive = True
|
||||
|
||||
@ -44,7 +44,7 @@ def get_version_type(version):
|
||||
if token in gentoo_types:
|
||||
types.append(token)
|
||||
if types:
|
||||
return types[0]
|
||||
return types[0] # TODO: consider returning all types
|
||||
return "release"
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user