euscanwww: Adding infos to EuscanResults

Now we store metadata, output from euscan is cleaned from colors
encoding when outputting json

Signed-off-by: volpino <fox91@anche.no>
This commit is contained in:
volpino 2012-06-26 18:13:28 +02:00
parent 699bb372a5
commit 8fdac1014c
4 changed files with 207 additions and 7 deletions

View File

@ -0,0 +1,191 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'EuscanResult.scan_time'
db.add_column('djeuscan_euscanresult', 'scan_time',
self.gf('django.db.models.fields.FloatField')(null=True, blank=True),
keep_default=False)
# Adding field 'EuscanResult.ebuild'
db.add_column('djeuscan_euscanresult', 'ebuild',
self.gf('django.db.models.fields.CharField')(default='', max_length=256, blank=True),
keep_default=False)
def backwards(self, orm):
# Deleting field 'EuscanResult.scan_time'
db.delete_column('djeuscan_euscanresult', 'scan_time')
# Deleting field 'EuscanResult.ebuild'
db.delete_column('djeuscan_euscanresult', 'ebuild')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'djeuscan.categoryassociation': {
'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.categorylog': {
'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.euscanresult': {
'Meta': {'object_name': 'EuscanResult'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'})
},
'djeuscan.herd': {
'Meta': {'object_name': 'Herd'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'djeuscan.herdassociation': {
'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.herdlog': {
'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.log': {
'Meta': {'object_name': 'Log'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'djeuscan.maintainer': {
'Meta': {'object_name': 'Maintainer'},
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.maintainerassociation': {
'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.maintainerlog': {
'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
},
'djeuscan.package': {
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.packageassociation': {
'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'djeuscan.refreshpackagequery': {
'Meta': {'object_name': 'RefreshPackageQuery'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'query': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'})
},
'djeuscan.version': {
'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'version_type': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
},
'djeuscan.versionlog': {
'Meta': {'object_name': 'VersionLog'},
'action': ('django.db.models.fields.IntegerField', [], {}),
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.worldlog': {
'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
}
}
complete_apps = ['djeuscan']

View File

@ -187,6 +187,9 @@ class EuscanResult(models.Model):
datetime = models.DateTimeField() datetime = models.DateTimeField()
result = models.TextField(blank=True) result = models.TextField(blank=True)
scan_time = models.FloatField(null=True, blank=True)
ebuild = models.CharField(blank=True, max_length=256)
objects = EuscanResultManager() objects = EuscanResultManager()
class Meta: class Meta:

View File

@ -25,26 +25,31 @@ class ScanUpstream(object):
try: try:
cpv = out[package]["metadata"]["cpv"] cpv = out[package]["metadata"]["cpv"]
scan_time = out[package]["metadata"]["scan_time"]
ebuild = out[package]["metadata"]["ebuild"]
except KeyError: except KeyError:
return {} return {}
with commit_on_success():
obj = self.store_package(cpv) obj = self.store_package(cpv)
for res in out[package]["result"]: for res in out[package]["result"]:
self.store_version(obj, res["version"], " ".join(res["urls"])) self.store_version(obj, res["version"], " ".join(res["urls"]))
self.store_result(obj, out_json) self.store_result(obj, out_json, scan_time, ebuild)
return out return out
def store_result(self, package, log): def store_result(self, package, formatted_log, scan_time, ebuild):
# Remove previous logs # Remove previous logs
EuscanResult.objects.filter(package=package).delete() EuscanResult.objects.filter(package=package).delete()
obj = EuscanResult() obj = EuscanResult()
obj.package = package obj.package = package
obj.result = log obj.result = formatted_log
obj.datetime = timezone.now() obj.datetime = timezone.now()
obj.scan_time = scan_time
obj.ebuild = ebuild
obj.save() obj.save()
def store_package(self, cpv): def store_package(self, cpv):

View File

@ -3,6 +3,7 @@ from collections import defaultdict
import json import json
import signal import signal
import time import time
import re
from gentoolkit import pprinter as pp from gentoolkit import pprinter as pp
import portage import portage
@ -61,7 +62,7 @@ class EOutputMem(EOutput):
self.out = StringIO() self.out = StringIO()
def getvalue(self): def getvalue(self):
return self.out.getvalue() return re.sub("\033\[[0-9;]+m", "", self.out.getvalue())
def _write(self, f, msg): def _write(self, f, msg):
super(EOutputMem, self)._write(self.out, msg) super(EOutputMem, self)._write(self.out, msg)