src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'fox/master'

Browse Source 
Conflicts:
	euscanwww/djeuscan/helpers.py

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>
This commit is contained in:
Corentin Chary 2012-07-26 21:17:10 +02:00
commit 29b1ad557a
26 changed files with 992 additions and 376 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
bin/euscan_patch_metadata
View File

@ -9,7 +9,7 @@ import tarfile
import gzip import gzip
import logging import logging
import shutil import shutil
import subprocess from difflib import unified_diff
from portage.exception import AmbiguousPackageName from portage.exception import AmbiguousPackageName
from gentoolkit.query import Query from gentoolkit.query import Query
@ -117,7 +117,7 @@ def get_deb_url(name):
content = opened.read() content = opened.read()
for link in BeautifulSoup(content, parseOnlyThese=SoupStrainer("a")): for link in BeautifulSoup(content, parseOnlyThese=SoupStrainer("a")):
if re.match("[^\s]+\.debian\.tar\.gz", link.text): if re.match("[^\s]+\.debian\.tar\.(?:gz|bz2)", link.text):
deb_url = link["href"] deb_url = link["href"]
deb_type = "source" deb_type = "source"
break break
@ -133,9 +133,11 @@ def get_deb_url(name):
return deb_url, deb_type return deb_url, deb_type
def patch_metadata(metadata_path, watch_data, diff=False): def patch_metadata(package, watch_data, diff=False):
logger.info(" Patching metadata file") logger.info(" Patching metadata file")
metadata_path = package.metadata.metadata_path
with open(metadata_path) as fp: with open(metadata_path) as fp:
original = fp.read() original = fp.read()
rindent, indent = guess_indent_values(original) rindent, indent = guess_indent_values(original)
@ -203,14 +205,17 @@ def patch_metadata(metadata_path, watch_data, diff=False):
if not diff: if not diff:
return data return data
else: else:
_, data_path = mkstemp() # Generate clean a/category/package/metadata.xml path
with open(data_path, "w") as f: n = metadata_path.find(package.category)
f.write(data) if n != -1:
sub = subprocess.Popen(["diff", metadata_path, data_path], metadata_path = metadata_path[n:]
stdout=subprocess.PIPE) res = unified_diff(
content = sub.stdout.read() original.splitlines(True),
os.unlink(data_path) data.splitlines(True),
return content fromfile=os.path.join('a/', metadata_path),
tofile=os.path.join('b/', metadata_path),
)
return "".join([x for x in res])
def process_package(query, diff=False): def process_package(query, diff=False):
@ -236,12 +241,11 @@ def process_package(query, diff=False):
if '9999' in package.version and len(matches) > 0: if '9999' in package.version and len(matches) > 0:
package = matches.pop() package = matches.pop()
metadata_path = package.metadata.metadata_path
watch_data = get_watch_data(package) watch_data = get_watch_data(package)
if watch_data is None: if watch_data is None:
logger.error(" No watch file found") logger.error(" No watch file found")
else: else:
return patch_metadata(metadata_path, watch_data, diff=diff) return patch_metadata(package, watch_data, diff=diff)
def main(): def main():
@ -261,7 +265,7 @@ def main():
logger.info("Processing %s..." % package) logger.info("Processing %s..." % package)
result = process_package(package, opts.diff) result = process_package(package, opts.diff)
if result: if result:
print result sys.stdout.write(result)
if __name__ == "__main__": if __name__ == "__main__":
main() main()

9
euscanwww/djeuscan/admin.py
View File

@ -1,7 +1,6 @@
from djeuscan.models import Package, Version, VersionLog, EuscanResult, \ from djeuscan.models import Package, Version, VersionLog, EuscanResult, \
Log, WorldLog, CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer, \ Log, WorldLog, CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer, \
RefreshPackageQuery, HerdAssociation, CategoryAssociation, \ RefreshPackageQuery, Category, Overlay, ProblemReport
MaintainerAssociation, PackageAssociation, ProblemReport
from django.contrib import admin from django.contrib import admin
@ -54,9 +53,7 @@ admin.site.register(HerdLog)
admin.site.register(MaintainerLog) admin.site.register(MaintainerLog)
admin.site.register(RefreshPackageQuery) admin.site.register(RefreshPackageQuery)
admin.site.register(HerdAssociation) admin.site.register(Category)
admin.site.register(CategoryAssociation) admin.site.register(Overlay)
admin.site.register(MaintainerAssociation)
admin.site.register(PackageAssociation)
admin.site.register(ProblemReport, ProblemReportAdmin) admin.site.register(ProblemReport, ProblemReportAdmin)

30
euscanwww/djeuscan/helpers.py
View File

@ -76,33 +76,33 @@ class catch_and_return(object):
return wrapper return wrapper
def get_account_categories(user): def get_profile(user):
from djeuscan.models import Package, CategoryAssociation from djeuscan.models import UserProfile
try:
return user.get_profile()
except UserProfile.DoesNotExist:
UserProfile.objects.create(user=user)
return user.get_profile()
def get_account_categories(user):
from djeuscan.models import Package
# TODO: This is quite ugly # TODO: This is quite ugly
category_names = [obj.category for obj in category_names = [obj.name for obj in get_profile(user).categories.all()]
CategoryAssociation.objects.filter(user=user)]
return [c for c in Package.objects.categories() return [c for c in Package.objects.categories()
if c["category"] in category_names] if c["category"] in category_names]
def get_account_herds(user): def get_account_herds(user):
from djeuscan.models import Package, HerdAssociation from djeuscan.models import Package
ids = [obj.herd.pk for obj in ids = [herd.pk for herd in get_profile(user).herds.all()]
HerdAssociation.objects.filter(user=user)]
return Package.objects.herds(ids=ids) return Package.objects.herds(ids=ids)
def get_account_maintainers(user): def get_account_maintainers(user):
from djeuscan.models import Package, MaintainerAssociation from djeuscan.models import Package
ids = [obj.maintainer.pk for obj in ids = [obj.pk for obj in get_profile(user).maintainers.all()]
MaintainerAssociation.objects.filter(user=user)]
return Package.objects.maintainers(ids=ids) return Package.objects.maintainers(ids=ids)
def get_account_packages(user):
from djeuscan.models import PackageAssociation
return [obj.package for obj in
PackageAssociation.objects.filter(user=user)]

355
euscanwww/djeuscan/migrations/0016_auto__del_maintainerassociation__del_unique_maintainerassociation_user.py Normal file
View File

@ -0,0 +1,355 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Removing unique constraint on 'CategoryAssociation', fields ['user', 'category']
db.delete_unique('djeuscan_categoryassociation', ['user_id', 'category'])
# Removing unique constraint on 'OverlayAssociation', fields ['user', 'overlay']
db.delete_unique('djeuscan_overlayassociation', ['user_id', 'overlay'])
# Removing unique constraint on 'HerdAssociation', fields ['user', 'herd']
db.delete_unique('djeuscan_herdassociation', ['user_id', 'herd_id'])
# Removing unique constraint on 'PackageAssociation', fields ['user', 'package']
db.delete_unique('djeuscan_packageassociation', ['user_id', 'package_id'])
# Removing unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer']
db.delete_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id'])
# Deleting model 'MaintainerAssociation'
db.delete_table('djeuscan_maintainerassociation')
# Deleting model 'PackageAssociation'
db.delete_table('djeuscan_packageassociation')
# Deleting model 'HerdAssociation'
db.delete_table('djeuscan_herdassociation')
# Deleting model 'OverlayAssociation'
db.delete_table('djeuscan_overlayassociation')
# Deleting model 'CategoryAssociation'
db.delete_table('djeuscan_categoryassociation')
# Adding model 'UserProfile'
db.create_table('djeuscan_userprofile', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('user', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['auth.User'], unique=True)),
))
db.send_create_signal('djeuscan', ['UserProfile'])
# Adding M2M table for field herds on 'UserProfile'
db.create_table('djeuscan_userprofile_herds', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)),
('herd', models.ForeignKey(orm['djeuscan.herd'], null=False))
))
db.create_unique('djeuscan_userprofile_herds', ['userprofile_id', 'herd_id'])
# Adding M2M table for field maintainers on 'UserProfile'
db.create_table('djeuscan_userprofile_maintainers', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)),
('maintainer', models.ForeignKey(orm['djeuscan.maintainer'], null=False))
))
db.create_unique('djeuscan_userprofile_maintainers', ['userprofile_id', 'maintainer_id'])
# Adding M2M table for field packages on 'UserProfile'
db.create_table('djeuscan_userprofile_packages', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)),
('package', models.ForeignKey(orm['djeuscan.package'], null=False))
))
db.create_unique('djeuscan_userprofile_packages', ['userprofile_id', 'package_id'])
# Adding M2M table for field categories on 'UserProfile'
db.create_table('djeuscan_userprofile_categories', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)),
('category', models.ForeignKey(orm['djeuscan.category'], null=False))
))
db.create_unique('djeuscan_userprofile_categories', ['userprofile_id', 'category_id'])
# Adding M2M table for field overlays on 'UserProfile'
db.create_table('djeuscan_userprofile_overlays', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)),
('overlay', models.ForeignKey(orm['djeuscan.overlay'], null=False))
))
db.create_unique('djeuscan_userprofile_overlays', ['userprofile_id', 'overlay_id'])
# Adding model 'Category'
db.create_table('djeuscan_category', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('name', self.gf('django.db.models.fields.CharField')(max_length=128)),
))
db.send_create_signal('djeuscan', ['Category'])
# Adding model 'Overlay'
db.create_table('djeuscan_overlay', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('name', self.gf('django.db.models.fields.CharField')(max_length=128)),
))
db.send_create_signal('djeuscan', ['Overlay'])
def backwards(self, orm):
# Adding model 'MaintainerAssociation'
db.create_table('djeuscan_maintainerassociation', (
('maintainer', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Maintainer'])),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
))
db.send_create_signal('djeuscan', ['MaintainerAssociation'])
# Adding unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer']
db.create_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id'])
# Adding model 'PackageAssociation'
db.create_table('djeuscan_packageassociation', (
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('package', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Package'])),
))
db.send_create_signal('djeuscan', ['PackageAssociation'])
# Adding unique constraint on 'PackageAssociation', fields ['user', 'package']
db.create_unique('djeuscan_packageassociation', ['user_id', 'package_id'])
# Adding model 'HerdAssociation'
db.create_table('djeuscan_herdassociation', (
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('herd', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Herd'])),
))
db.send_create_signal('djeuscan', ['HerdAssociation'])
# Adding unique constraint on 'HerdAssociation', fields ['user', 'herd']
db.create_unique('djeuscan_herdassociation', ['user_id', 'herd_id'])
# Adding model 'OverlayAssociation'
db.create_table('djeuscan_overlayassociation', (
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('overlay', self.gf('django.db.models.fields.CharField')(max_length=128)),
))
db.send_create_signal('djeuscan', ['OverlayAssociation'])
# Adding unique constraint on 'OverlayAssociation', fields ['user', 'overlay']
db.create_unique('djeuscan_overlayassociation', ['user_id', 'overlay'])
# Adding model 'CategoryAssociation'
db.create_table('djeuscan_categoryassociation', (
('category', self.gf('django.db.models.fields.CharField')(max_length=128)),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
))
db.send_create_signal('djeuscan', ['CategoryAssociation'])
# Adding unique constraint on 'CategoryAssociation', fields ['user', 'category']
db.create_unique('djeuscan_categoryassociation', ['user_id', 'category'])
# Deleting model 'UserProfile'
db.delete_table('djeuscan_userprofile')
# Removing M2M table for field herds on 'UserProfile'
db.delete_table('djeuscan_userprofile_herds')
# Removing M2M table for field maintainers on 'UserProfile'
db.delete_table('djeuscan_userprofile_maintainers')
# Removing M2M table for field packages on 'UserProfile'
db.delete_table('djeuscan_userprofile_packages')
# Removing M2M table for field categories on 'UserProfile'
db.delete_table('djeuscan_userprofile_categories')
# Removing M2M table for field overlays on 'UserProfile'
db.delete_table('djeuscan_userprofile_overlays')
# Deleting model 'Category'
db.delete_table('djeuscan_category')
# Deleting model 'Overlay'
db.delete_table('djeuscan_overlay')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'djeuscan.category': {
'Meta': {'object_name': 'Category'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.categorylog': {
'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.euscanresult': {
'Meta': {'object_name': 'EuscanResult'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'})
},
'djeuscan.herd': {
'Meta': {'object_name': 'Herd'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'})
},
'djeuscan.herdlog': {
'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.log': {
'Meta': {'object_name': 'Log'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'djeuscan.maintainer': {
'Meta': {'object_name': 'Maintainer'},
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.maintainerlog': {
'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
},
'djeuscan.overlay': {
'Meta': {'object_name': 'Overlay'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.package': {
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.problemreport': {
'Meta': {'object_name': 'ProblemReport'},
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'message': ('django.db.models.fields.TextField', [], {}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'subject': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Version']", 'null': 'True', 'blank': 'True'})
},
'djeuscan.refreshpackagequery': {
'Meta': {'object_name': 'RefreshPackageQuery'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'symmetrical': 'False'})
},
'djeuscan.userprofile': {
'Meta': {'object_name': 'UserProfile'},
'categories': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Category']", 'symmetrical': 'False'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'}),
'overlays': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Overlay']", 'symmetrical': 'False'}),
'packages': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Package']", 'symmetrical': 'False'}),
'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'})
},
'djeuscan.version': {
'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'ebuild_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'metadata_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
},
'djeuscan.versionlog': {
'Meta': {'object_name': 'VersionLog'},
'action': ('django.db.models.fields.IntegerField', [], {}),
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
},
'djeuscan.worldlog': {
'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
}
}
complete_apps = ['djeuscan']

197
euscanwww/djeuscan/migrations/0017_auto__add_unique_overlay_name__add_unique_category_name.py Normal file
View File

@ -0,0 +1,197 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding unique constraint on 'Overlay', fields ['name']
db.create_unique('djeuscan_overlay', ['name'])
# Adding unique constraint on 'Category', fields ['name']
db.create_unique('djeuscan_category', ['name'])
def backwards(self, orm):
# Removing unique constraint on 'Category', fields ['name']
db.delete_unique('djeuscan_category', ['name'])
# Removing unique constraint on 'Overlay', fields ['name']
db.delete_unique('djeuscan_overlay', ['name'])
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'djeuscan.category': {
'Meta': {'object_name': 'Category'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'})
},
'djeuscan.categorylog': {
'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.euscanresult': {
'Meta': {'object_name': 'EuscanResult'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'})
},
'djeuscan.herd': {
'Meta': {'object_name': 'Herd'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'})
},
'djeuscan.herdlog': {
'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.log': {
'Meta': {'object_name': 'Log'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'djeuscan.maintainer': {
'Meta': {'object_name': 'Maintainer'},
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.maintainerlog': {
'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
},
'djeuscan.overlay': {
'Meta': {'object_name': 'Overlay'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'})
},
'djeuscan.package': {
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.problemreport': {
'Meta': {'object_name': 'ProblemReport'},
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'message': ('django.db.models.fields.TextField', [], {}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'subject': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Version']", 'null': 'True', 'blank': 'True'})
},
'djeuscan.refreshpackagequery': {
'Meta': {'object_name': 'RefreshPackageQuery'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'symmetrical': 'False'})
},
'djeuscan.userprofile': {
'Meta': {'object_name': 'UserProfile'},
'categories': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Category']", 'symmetrical': 'False'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'}),
'overlays': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Overlay']", 'symmetrical': 'False'}),
'packages': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Package']", 'symmetrical': 'False'}),
'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'})
},
'djeuscan.version': {
'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'ebuild_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'metadata_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
},
'djeuscan.versionlog': {
'Meta': {'object_name': 'VersionLog'},
'action': ('django.db.models.fields.IntegerField', [], {}),
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
},
'djeuscan.worldlog': {
'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
}
}
complete_apps = ['djeuscan']

80
euscanwww/djeuscan/models.py
View File

@ -231,6 +231,31 @@ class EuscanResult(models.Model):
) )
class Category(models.Model):
name = models.CharField(max_length=128, validators=[validate_category],
unique=True)
def __unicode__(self):
return self.name
class Overlay(models.Model):
name = models.CharField(max_length=128, validators=[validate_name],
unique=True)
def __unicode__(self):
return self.name
class UserProfile(models.Model):
user = models.OneToOneField(User)
herds = models.ManyToManyField(Herd)
maintainers = models.ManyToManyField(Maintainer)
packages = models.ManyToManyField(Package)
categories = models.ManyToManyField(Category)
overlays = models.ManyToManyField(Overlay)
class Log(models.Model): class Log(models.Model):
""" """
Model used for keeping data for charts Model used for keeping data for charts
@ -303,61 +328,6 @@ class RefreshPackageQuery(models.Model):
return u'[%d] %s' % (self.priority, self.package) return u'[%d] %s' % (self.priority, self.package)
class HerdAssociation(models.Model):
user = models.ForeignKey(User)
herd = models.ForeignKey(Herd)
class Meta:
unique_together = ['user', 'herd']
def __unicode__(self):
return u'[%s] %s' % (self.user, self.herd)
class MaintainerAssociation(models.Model):
user = models.ForeignKey(User)
maintainer = models.ForeignKey(Maintainer)
class Meta:
unique_together = ['user', 'maintainer']
def __unicode__(self):
return u'[%s] %s' % (self.user, self.maintainer)
class PackageAssociation(models.Model):
user = models.ForeignKey(User)
package = models.ForeignKey(Package)
class Meta:
unique_together = ['user', 'package']
def __unicode__(self):
return u'[%s] %s' % (self.user, self.package)
class CategoryAssociation(models.Model):
user = models.ForeignKey(User)
category = models.CharField(max_length=128, validators=[validate_category])
class Meta:
unique_together = ['user', 'category']
def __unicode__(self):
return u'[%s] %s' % (self.user, self.category)
class OverlayAssociation(models.Model):
user = models.ForeignKey(User)
overlay = models.CharField(max_length=128, validators=[validate_name])
class Meta:
unique_together = ['user', 'overlay']
def __unicode__(self):
return u'[%s] %s' % (self.user, self.category)
class ProblemReport(models.Model): class ProblemReport(models.Model):
package = models.ForeignKey(Package) package = models.ForeignKey(Package)
version = models.ForeignKey(Version, null=True, blank=True) version = models.ForeignKey(Version, null=True, blank=True)

23
euscanwww/djeuscan/processing/misc/update_counters.py
View File

@ -1,8 +1,8 @@
from django.db.transaction import commit_on_success from django.db.transaction import commit_on_success
from django.utils import timezone from django.utils import timezone
from djeuscan.models import Package, Herd, Maintainer, Version from djeuscan.models import Package, Herd, Maintainer, Version, HerdLog, \
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog MaintainerLog, CategoryLog, WorldLog, Category, Overlay
from djeuscan import charts from djeuscan import charts
from djeuscan.processing import FakeLogger from djeuscan.processing import FakeLogger
@ -48,6 +48,19 @@ def update_counters(fast=False, nolog=False, logger=None):
wlog = None wlog = None
# Populate Category and Overlay
for cat in Package.objects.values('category').distinct():
obj, created = Category.objects.get_or_create(name=cat["category"])
if created:
logger.info("+ [c] %s", cat["category"])
for overlay in Version.objects.values('overlay').distinct():
if not overlay["overlay"]:
continue
obj, created = Overlay.objects.get_or_create(name=overlay["overlay"])
if created:
logger.info("+ [o] %s", overlay["overlay"])
if not nolog: if not nolog:
wlog = WorldLog() wlog = WorldLog()
wlog.datetime = now wlog.datetime = now
@ -157,17 +170,17 @@ def update_counters(fast=False, nolog=False, logger=None):
return return
for clog in categories.values(): for clog in categories.values():
logger.info('+ [cl] %s' % clog) logger.info('+ [cl] %s', clog)
charts.rrd_update('category-%s' % clog.category, now, clog) charts.rrd_update('category-%s' % clog.category, now, clog)
clog.save() clog.save()
for hlog in herds.values(): for hlog in herds.values():
logger.info('+ [hl] %s' % hlog) logger.info('+ [hl] %s', hlog)
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog) charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
hlog.save() hlog.save()
for mlog in maintainers.values(): for mlog in maintainers.values():
logger.info('+ [ml] %s' % mlog) logger.info('+ [ml] %s', mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog) charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save() mlog.save()

39
euscanwww/djeuscan/tests/euscan_factory.py
View File

@ -4,10 +4,12 @@ from datetime import datetime
from collections import defaultdict from collections import defaultdict
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.db import IntegrityError
import factory import factory
from djeuscan.models import Herd, Maintainer, Package, Version, EuscanResult from djeuscan.models import Herd, Maintainer, Package, Version, EuscanResult, \
Category, Overlay
class UserFactory(factory.Factory): class UserFactory(factory.Factory):
@ -36,6 +38,18 @@ class MaintainerFactory(factory.Factory):
email = factory.LazyAttribute(lambda a: "%s@example.com" % a.name) email = factory.LazyAttribute(lambda a: "%s@example.com" % a.name)
class CategoryFactory(factory.Factory):
FACTORY_FOR = Category
name = factory.LazyAttribute(lambda a: random_string())
class OverlayFactory(factory.Factory):
FACTORY_FOR = Overlay
name = factory.LazyAttribute(lambda a: random_string())
class PackageFactory(factory.Factory): class PackageFactory(factory.Factory):
FACTORY_FOR = Package FACTORY_FOR = Package
@ -46,6 +60,16 @@ class PackageFactory(factory.Factory):
description = "This is a test package" description = "This is a test package"
homepage = "http://testpackage.com" homepage = "http://testpackage.com"
@classmethod
def _prepare(cls, create, **kwargs):
package = super(PackageFactory, cls)._prepare(create, **kwargs)
category = kwargs.pop('category', None)
if create:
CategoryFactory.create(name=category)
else:
CategoryFactory.build(name=category)
return package
class VersionFactory(factory.Factory): class VersionFactory(factory.Factory):
FACTORY_FOR = Version FACTORY_FOR = Version
@ -59,6 +83,19 @@ class VersionFactory(factory.Factory):
urls = "http://packageurl.com" urls = "http://packageurl.com"
alive = True alive = True
@classmethod
def _prepare(cls, create, **kwargs):
version = super(VersionFactory, cls)._prepare(create, **kwargs)
overlay = kwargs.pop('overlay', None)
try:
if create:
OverlayFactory.create(name=overlay)
else:
OverlayFactory.build(name=overlay)
except IntegrityError:
pass
return version
class EuscanResultFactory(factory.Factory): class EuscanResultFactory(factory.Factory):
FACTORY_FOR = EuscanResult FACTORY_FOR = EuscanResult

42
euscanwww/djeuscan/tests/test_views.py
View File

@ -5,13 +5,12 @@ try:
except ImportError: except ImportError:
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from djeuscan.helpers import get_profile
from djeuscan.tests import SystemTestCase from djeuscan.tests import SystemTestCase
from djeuscan.tests.euscan_factory import PackageFactory, setup_maintainers, \ from djeuscan.tests.euscan_factory import PackageFactory, setup_maintainers, \
setup_herds, setup_categories, setup_overlays setup_herds, setup_categories, setup_overlays
from djeuscan.models import PackageAssociation, CategoryAssociation, \
HerdAssociation, MaintainerAssociation
class PagesTest(SystemTestCase): class PagesTest(SystemTestCase):
""" """
@ -50,11 +49,10 @@ class PackageTests(SystemTestCase):
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
def test_favourite(self): def test_favourite(self):
self.assertEqual(PackageAssociation.objects.count(), 0)
response = self.get("package", category=self.package.category, response = self.get("package", category=self.package.category,
package=self.package.name) package=self.package.name)
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
self.assertNotIn("Watch", response.content) self.assertNotIn("Watch", response.content)
with self.login(): with self.login():
@ -62,11 +60,14 @@ class PackageTests(SystemTestCase):
package=self.package.name) package=self.package.name)
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
user = response.context["user"]
self.assertEquals(get_profile(user).categories.count(), 0)
self.assertIn("Watch", response.content) self.assertIn("Watch", response.content)
self.post("favourite_package", category=self.package.category, self.post("favourite_package", category=self.package.category,
package=self.package.name) package=self.package.name)
self.assertEqual(PackageAssociation.objects.count(), 1) self.assertEquals(get_profile(user).packages.count(), 1)
response = self.get("accounts_packages") response = self.get("accounts_packages")
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
@ -75,7 +76,7 @@ class PackageTests(SystemTestCase):
self.post("unfavourite_package", category=self.package.category, self.post("unfavourite_package", category=self.package.category,
package=self.package.name) package=self.package.name)
self.assertEqual(PackageAssociation.objects.count(), 0) self.assertEquals(get_profile(user).packages.count(), 0)
class SectionTests(SystemTestCase): class SectionTests(SystemTestCase):
@ -118,9 +119,9 @@ class CategoriesTests(SectionTests):
def test_favourite(self): def test_favourite(self):
category = self.categories[0] category = self.categories[0]
self.assertEqual(CategoryAssociation.objects.count(), 0)
response = self.get("category", category=category) response = self.get("category", category=category)
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
self.assertNotIn("Watch", response.content) self.assertNotIn("Watch", response.content)
@ -128,10 +129,13 @@ class CategoriesTests(SectionTests):
response = self.get("category", category=category) response = self.get("category", category=category)
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
user = response.context["user"]
self.assertEquals(get_profile(user).categories.count(), 0)
self.assertIn("Watch", response.content) self.assertIn("Watch", response.content)
self.post("favourite_category", category=category) self.post("favourite_category", category=category)
self.assertEqual(CategoryAssociation.objects.count(), 1) self.assertEquals(get_profile(user).categories.count(), 1)
response = self.get("accounts_categories") response = self.get("accounts_categories")
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
@ -139,7 +143,8 @@ class CategoriesTests(SectionTests):
self._check_table(response, [category]) self._check_table(response, [category])
self.post("unfavourite_category", category=category) self.post("unfavourite_category", category=category)
self.assertEqual(CategoryAssociation.objects.count(), 0)
self.assertEquals(get_profile(user).categories.count(), 0)
class HerdsTests(SectionTests): class HerdsTests(SectionTests):
@ -167,7 +172,6 @@ class HerdsTests(SectionTests):
def test_favourite(self): def test_favourite(self):
herd = self.herds[0] herd = self.herds[0]
self.assertEqual(HerdAssociation.objects.count(), 0)
response = self.get("herd", herd=herd.herd) response = self.get("herd", herd=herd.herd)
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
@ -177,10 +181,13 @@ class HerdsTests(SectionTests):
response = self.get("herd", herd=herd.herd) response = self.get("herd", herd=herd.herd)
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
user = response.context["user"]
self.assertEquals(get_profile(user).herds.count(), 0)
self.assertIn("Watch", response.content) self.assertIn("Watch", response.content)
self.post("favourite_herd", herd=herd.herd) self.post("favourite_herd", herd=herd.herd)
self.assertEqual(HerdAssociation.objects.count(), 1) self.assertEquals(get_profile(user).herds.count(), 1)
response = self.get("accounts_herds") response = self.get("accounts_herds")
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
@ -188,7 +195,7 @@ class HerdsTests(SectionTests):
self._check_table(response, [herd], attr="herd") self._check_table(response, [herd], attr="herd")
self.post("unfavourite_herd", herd=herd.herd) self.post("unfavourite_herd", herd=herd.herd)
self.assertEqual(HerdAssociation.objects.count(), 0) self.assertEquals(get_profile(user).herds.count(), 0)
class MaintainersTests(SectionTests): class MaintainersTests(SectionTests):
@ -216,7 +223,6 @@ class MaintainersTests(SectionTests):
def test_favourite(self): def test_favourite(self):
maintainer = self.maintainers[0] maintainer = self.maintainers[0]
self.assertEqual(MaintainerAssociation.objects.count(), 0)
response = self.get("maintainer", maintainer_id=maintainer.pk) response = self.get("maintainer", maintainer_id=maintainer.pk)
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
@ -226,10 +232,13 @@ class MaintainersTests(SectionTests):
response = self.get("maintainer", maintainer_id=maintainer.pk) response = self.get("maintainer", maintainer_id=maintainer.pk)
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
user = response.context["user"]
self.assertEquals(get_profile(user).maintainers.count(), 0)
self.assertIn("Watch", response.content) self.assertIn("Watch", response.content)
self.post("favourite_maintainer", maintainer_id=maintainer.pk) self.post("favourite_maintainer", maintainer_id=maintainer.pk)
self.assertEqual(MaintainerAssociation.objects.count(), 1) self.assertEquals(get_profile(user).maintainers.count(), 1)
response = self.get("accounts_maintainers") response = self.get("accounts_maintainers")
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
@ -237,7 +246,8 @@ class MaintainersTests(SectionTests):
self._check_table(response, [maintainer], attr="name") self._check_table(response, [maintainer], attr="name")
self.post("unfavourite_maintainer", maintainer_id=maintainer.pk) self.post("unfavourite_maintainer", maintainer_id=maintainer.pk)
self.assertEqual(MaintainerAssociation.objects.count(), 0)
self.assertEquals(get_profile(user).maintainers.count(), 0)
class OverlayTests(SectionTests): class OverlayTests(SectionTests):

125
euscanwww/djeuscan/views.py
View File

@ -8,12 +8,10 @@ from django.shortcuts import get_object_or_404
from django.contrib.auth.decorators import login_required from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_POST from django.views.decorators.http import require_POST
from djeuscan.helpers import version_key, packages_from_names, \ from djeuscan.helpers import version_key, packages_from_names, get_profile, \
get_account_categories, get_account_herds, get_account_maintainers, \ get_account_categories, get_account_herds, get_account_maintainers
get_account_packages
from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \ from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
VersionLog, RefreshPackageQuery, HerdAssociation, MaintainerAssociation, \ VersionLog, RefreshPackageQuery, ProblemReport, Category, Overlay
CategoryAssociation, PackageAssociation, OverlayAssociation, ProblemReport
from djeuscan.forms import WorldForm, PackagesForm, ProblemReportForm from djeuscan.forms import WorldForm, PackagesForm, ProblemReportForm
from djeuscan.tasks import admin_tasks from djeuscan.tasks import admin_tasks
from djeuscan import charts from djeuscan import charts
@ -68,12 +66,8 @@ def category(request, category):
favourited = False favourited = False
if request.user.is_authenticated(): if request.user.is_authenticated():
try: if Category.objects.get(name=category) in \
CategoryAssociation.objects.get(user=request.user, get_profile(request.user).categories.all():
category=category)
except CategoryAssociation.DoesNotExist:
pass
else:
favourited = True favourited = True
return {'category': category, 'packages': packages, 'last_scan': last_scan, return {'category': category, 'packages': packages, 'last_scan': last_scan,
@ -104,11 +98,7 @@ def herd(request, herd):
favourited = False favourited = False
if request.user.is_authenticated(): if request.user.is_authenticated():
try: if herd in get_profile(request.user).herds.all():
HerdAssociation.objects.get(user=request.user, herd=herd)
except HerdAssociation.DoesNotExist:
pass
else:
favourited = True favourited = True
return {'herd': herd, 'packages': packages, "last_scan": last_scan, return {'herd': herd, 'packages': packages, "last_scan": last_scan,
@ -140,12 +130,7 @@ def maintainer(request, maintainer_id):
favourited = False favourited = False
if request.user.is_authenticated(): if request.user.is_authenticated():
try: if maintainer in get_profile(request.user).maintainers.all():
MaintainerAssociation.objects.get(user=request.user,
maintainer=maintainer)
except MaintainerAssociation.DoesNotExist:
pass
else:
favourited = True favourited = True
return {'maintainer': maintainer, 'packages': packages, return {'maintainer': maintainer, 'packages': packages,
@ -177,16 +162,14 @@ def overlay(request, overlay):
favourited = False favourited = False
if request.user.is_authenticated(): if request.user.is_authenticated():
try: if Overlay.objects.get(name=overlay) in \
OverlayAssociation.objects.get(user=request.user, overlay=overlay) get_profile(request.user).overlays.all():
except OverlayAssociation.DoesNotExist:
pass
else:
favourited = True favourited = True
return {'overlay': overlay, 'packages': packages, 'last_scan': last_scan, return {'overlay': overlay, 'packages': packages, 'last_scan': last_scan,
'favourited': favourited} 'favourited': favourited}
@render_to('euscan/package.html') @render_to('euscan/package.html')
def package(request, category, package): def package(request, category, package):
package = get_object_or_404(Package, category=category, name=package) package = get_object_or_404(Package, category=category, name=package)
@ -208,11 +191,7 @@ def package(request, category, package):
favourited = False favourited = False
if request.user.is_authenticated(): if request.user.is_authenticated():
try: if package in get_profile(request.user).packages.all():
PackageAssociation.objects.get(user=request.user, package=package)
except PackageAssociation.DoesNotExist:
pass
else:
favourited = True favourited = True
try: try:
@ -225,14 +204,15 @@ def package(request, category, package):
'package': package, 'package': package,
'packaged': packaged, 'packaged': packaged,
'upstream': upstream, 'upstream': upstream,
'log': log.messages(),
'vlog': vlog, 'vlog': vlog,
'log': log,
'msg': log.messages() if log else "", 'msg': log.messages() if log else "",
'last_scan': last_scan, 'last_scan': last_scan,
'favourited': favourited, 'favourited': favourited,
'refreshed': refreshed, 'refreshed': refreshed,
} }
@login_required @login_required
@render_to('euscan/problem.html') @render_to('euscan/problem.html')
def problem(request, category, package): def problem(request, category, package):
@ -268,6 +248,7 @@ def problem(request, category, package):
'msg': log.messages() if log else "", 'msg': log.messages() if log else "",
} }
@render_to('euscan/world.html') @render_to('euscan/world.html')
def world(request): def world(request):
world_form = WorldForm() world_form = WorldForm()
@ -394,9 +375,10 @@ def refresh_package(request, category, package):
@login_required @login_required
@render_to('euscan/accounts/index.html') @render_to('euscan/accounts/index.html')
def accounts_index(request): def accounts_index(request):
user = request.user
upstream_k = lambda c: c["n_versions"] - c["n_packaged"] upstream_k = lambda c: c["n_versions"] - c["n_packaged"]
categories = sorted(get_account_categories(request.user), categories = sorted(get_account_categories(user),
key=upstream_k, reverse=True) key=upstream_k, reverse=True)
c_upstream = sum([c["n_versions"] - c["n_packaged"] for c in categories]) c_upstream = sum([c["n_versions"] - c["n_packaged"] for c in categories])
herds = sorted(get_account_herds(request.user), herds = sorted(get_account_herds(request.user),
@ -405,7 +387,7 @@ def accounts_index(request):
maintainers = sorted(get_account_maintainers(request.user), maintainers = sorted(get_account_maintainers(request.user),
key=upstream_k, reverse=True) key=upstream_k, reverse=True)
m_upstream = sum([c["n_versions"] - c["n_packaged"] for c in maintainers]) m_upstream = sum([c["n_versions"] - c["n_packaged"] for c in maintainers])
packages = sorted(get_account_packages(request.user), packages = sorted(get_profile(user).packages.all(),
key=lambda p: p.n_versions - p.n_packaged, reverse=True) key=lambda p: p.n_versions - p.n_packaged, reverse=True)
p_upstream = sum([c.n_versions - c.n_packaged for c in packages]) p_upstream = sum([c.n_versions - c.n_packaged for c in packages])
return { return {
@ -437,14 +419,13 @@ def accounts_maintainers(request):
@login_required @login_required
@render_to('euscan/accounts/packages.html') @render_to('euscan/accounts/packages.html')
def accounts_packages(request): def accounts_packages(request):
return {"packages": get_account_packages(request.user)} return {"packages": get_profile(request.user).packages.all()}
@login_required @login_required
@render_to('euscan/accounts/overlays.html') @render_to('euscan/accounts/overlays.html')
def accounts_overlays(request): def accounts_overlays(request):
overlays = [obj.overlay for obj in overlays = [obj.name for obj in get_profile(request.user).overlays.all()]
OverlayAssociation.objects.filter(user=request.user)]
return {"overlays": overlays} return {"overlays": overlays}
@ -453,10 +434,8 @@ def accounts_overlays(request):
@ajax_request @ajax_request
def favourite_package(request, category, package): def favourite_package(request, category, package):
obj = get_object_or_404(Package, category=category, name=package) obj = get_object_or_404(Package, category=category, name=package)
_, created = PackageAssociation.objects.get_or_create( get_profile(request.user).packages.add(obj)
user=request.user, package=obj return {"success": True}
)
return {"success": created}
@login_required @login_required
@ -464,10 +443,7 @@ def favourite_package(request, category, package):
@ajax_request @ajax_request
def unfavourite_package(request, category, package): def unfavourite_package(request, category, package):
package = get_object_or_404(Package, category=category, name=package) package = get_object_or_404(Package, category=category, name=package)
obj = get_object_or_404( get_profile(request.user).packages.remove(package)
PackageAssociation, package=package, user=request.user
)
obj.delete()
return {"success": True} return {"success": True}
@ -476,10 +452,8 @@ def unfavourite_package(request, category, package):
@ajax_request @ajax_request
def favourite_herd(request, herd): def favourite_herd(request, herd):
obj = get_object_or_404(Herd, herd=herd) obj = get_object_or_404(Herd, herd=herd)
_, created = HerdAssociation.objects.get_or_create( get_profile(request.user).herds.add(obj)
user=request.user, herd=obj return {"success": True}
)
return {"success": created}
@login_required @login_required
@ -487,10 +461,7 @@ def favourite_herd(request, herd):
@ajax_request @ajax_request
def unfavourite_herd(request, herd): def unfavourite_herd(request, herd):
herd = get_object_or_404(Herd, herd=herd) herd = get_object_or_404(Herd, herd=herd)
obj = get_object_or_404( get_profile(request.user).herds.remove(herd)
HerdAssociation, herd=herd, user=request.user
)
obj.delete()
return {"success": True} return {"success": True}
@ -499,10 +470,8 @@ def unfavourite_herd(request, herd):
@ajax_request @ajax_request
def favourite_maintainer(request, maintainer_id): def favourite_maintainer(request, maintainer_id):
obj = get_object_or_404(Maintainer, pk=maintainer_id) obj = get_object_or_404(Maintainer, pk=maintainer_id)
_, created = MaintainerAssociation.objects.get_or_create( get_profile(request.user).maintainers.add(obj)
user=request.user, maintainer=obj return {"success": True}
)
return {"success": created}
@login_required @login_required
@ -510,10 +479,7 @@ def favourite_maintainer(request, maintainer_id):
@ajax_request @ajax_request
def unfavourite_maintainer(request, maintainer_id): def unfavourite_maintainer(request, maintainer_id):
maintainer = get_object_or_404(Maintainer, pk=maintainer_id) maintainer = get_object_or_404(Maintainer, pk=maintainer_id)
obj = get_object_or_404( get_profile(request.user).maintainers.remove(maintainer)
MaintainerAssociation, maintainer=maintainer, user=request.user
)
obj.delete()
return {"success": True} return {"success": True}
@ -521,25 +487,17 @@ def unfavourite_maintainer(request, maintainer_id):
@require_POST @require_POST
@ajax_request @ajax_request
def favourite_category(request, category): def favourite_category(request, category):
packages = Package.objects.for_category(category, last_versions=True) obj = Category.objects.get(name=category)
get_profile(request.user).categories.add(obj)
if not packages: return {"success": True}
raise Http404
_, created = CategoryAssociation.objects.get_or_create(
user=request.user, category=category
)
return {"success": created}
@login_required @login_required
@require_POST @require_POST
@ajax_request @ajax_request
def unfavourite_category(request, category): def unfavourite_category(request, category):
obj = get_object_or_404( obj = Category.objects.get(name=category)
CategoryAssociation, user=request.user, category=category get_profile(request.user).categories.remove(obj)
)
obj.delete()
return {"success": True} return {"success": True}
@ -547,22 +505,15 @@ def unfavourite_category(request, category):
@require_POST @require_POST
@ajax_request @ajax_request
def favourite_overlay(request, overlay): def favourite_overlay(request, overlay):
packages = Package.objects.for_overlay(overlay) obj = Category.objects.get(name=overlay)
if not packages: get_profile(request.user).overlays.add(obj)
raise Http404 return {"success": True}
_, created = OverlayAssociation.objects.get_or_create(
user=request.user, overlay=overlay
)
return {"success": created}
@login_required @login_required
@require_POST @require_POST
@ajax_request @ajax_request
def unfavourite_overlay(request, overlay): def unfavourite_overlay(request, overlay):
obj = get_object_or_404( obj = Category.objects.get(name=overlay)
OverlayAssociation, user=request.user, overlay=overlay get_profile(request.user).overlays.remove(obj)
)
obj.delete()
return {"success": True} return {"success": True}

2
euscanwww/euscanwww/settings.py
View File

@ -261,3 +261,5 @@ except ImportError, ex:
os.environ['ROOT'] = PORTAGE_ROOT os.environ['ROOT'] = PORTAGE_ROOT
os.environ['PORTAGE_CONFIGROOT'] = PORTAGE_CONFIGROOT os.environ['PORTAGE_CONFIGROOT'] = PORTAGE_CONFIGROOT
os.environ['EIX_CACHEFILE'] = EIX_CACHEFILE os.environ['EIX_CACHEFILE'] = EIX_CACHEFILE
AUTH_PROFILE_MODULE = 'djeuscan.UserProfile'

1
euscanwww/runtests.py
View File

@ -29,6 +29,7 @@ settings.configure(
USE_TZ=True, USE_TZ=True,
TASKS_CONCURRENTLY=8, TASKS_CONCURRENTLY=8,
TASKS_SUBTASK_PACKAGES=32, TASKS_SUBTASK_PACKAGES=32,
AUTH_PROFILE_MODULE="djeuscan.UserProfile"
) )

103
pym/euscan/handlers/__init__.py
View File

@ -1,35 +1,86 @@
import pkgutil import sys
from euscan import CONFIG, output
# autoimport all modules in this directory and append them to handlers list from euscan.handlers.package import handlers as pkg_handlers
handlers = [] from euscan.handlers.url import handlers as url_handlers
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
module = loader.find_module(module_name).load_module(module_name)
handlers.append(module)
# sort handlers by priority (e.g.: generic should be run lastly)
handlers = sorted(
handlers,
key=lambda handler: handler.PRIORITY,
reverse=True
)
def find_best_handler(pkg, url): def find_best_pkg_handler(pkg):
for handler in handlers: """
Find the best handler for the given package
"""
for handler in pkg_handlers:
if handler.can_handle(pkg):
return handler
return None
def find_best_url_handler(pkg, url):
"""
Find the best handler for the given url
"""
for handler in url_handlers:
if handler.can_handle(pkg, url): if handler.can_handle(pkg, url):
return handler return handler
return None return None
def scan(pkg, url): def scan(pkg, urls, on_progress=None):
handler = find_best_handler(pkg, url) """
if handler: Scans upstream for the given package.
return handler.scan(pkg, url) First tries if a package wide handler is available, then fallbacks
return [] in url handling.
"""
pkg_handler = find_best_pkg_handler(pkg)
if pkg_handler:
if on_progress:
on_progress(increment=35)
if not CONFIG['quiet'] and not CONFIG['format']:
sys.stdout.write("\n")
def brute_force(pkg, url): versions = pkg_handler.scan(pkg)
handler = find_best_handler(pkg, url)
if handler: if on_progress:
return handler.brute_force(pkg, url) on_progress(increment=35)
return [] return versions
if on_progress:
progress_available = 70
num_urls = sum([len(urls[fn]) for fn in urls])
if num_urls > 0:
progress_increment = progress_available / num_urls
else:
progress_increment = 0
versions = []
for filename in urls:
for url in urls[filename]:
if on_progress and progress_available > 0:
on_progress(increment=progress_increment)
progress_available -= progress_increment
if not CONFIG['quiet'] and not CONFIG['format']:
sys.stdout.write("\n")
output.einfo("SRC_URI is '%s'" % url)
if '://' not in url:
output.einfo("Invalid url '%s'" % url)
continue
try:
url_handler = find_best_url_handler(pkg, url)
versions.extend(url_handler.scan(pkg, url))
except Exception as e:
output.ewarn(
"Handler failed: [%s] %s" %
(e.__class__.__name__, e.message)
)
if versions and CONFIG['oneshot']:
break
if on_progress and progress_available > 0:
on_progress(increment=progress_available)
return versions

19
pym/euscan/handlers/package/__init__.py Normal file
View File

@ -0,0 +1,19 @@
"""
Package wide handlers for scanning upstream
"""
import pkgutil
handlers = []
# autoimport all modules in this directory and append them to handlers list
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
module = loader.find_module(module_name).load_module(module_name)
handlers.append(module)
# sort handlers by priority
handlers = sorted(
handlers,
key=lambda handler: handler.PRIORITY,
reverse=True
)

43
pym/euscan/handlers/package/remote_id.py Normal file
View File

@ -0,0 +1,43 @@
from euscan.handlers.url import handlers
from euscan import output
PRIORITY = 100
HANDLER_NAME = "remote_id"
CONFIDENCE = 100.0
url_handlers = {handler.HANDLER_NAME: handler for handler in handlers}
def can_handle(pkg):
# Return True if there's at least one remote-id that can be
# handled by euscan
try:
remoteids = pkg.metadata.upstream()[0].upstream_remoteids()
except IndexError:
pass
else:
if len(remoteids) > 0:
for remote_value, remote_type in remoteids:
if remote_type in url_handlers:
return True
return False
def scan(pkg):
output.einfo("Using remote-id data")
ret = []
remoteids = pkg.metadata.upstream()[0].upstream_remoteids()
for remote_value, remote_type in remoteids:
if remote_type in url_handlers:
remote_data = remote_value.split("/")
scan_remote = getattr(
url_handlers[remote_type], "scan_remote", None
)
if scan_remote:
for url, pv in scan_remote(pkg, remote_data):
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret

14
pym/euscan/handlers/watch.py → pym/euscan/handlers/package/watch.py
View File

@ -3,7 +3,7 @@ import urllib2
import portage import portage
from euscan.handlers import generic from euscan.handlers.url import generic
from euscan import output, helpers from euscan import output, helpers
PRIORITY = 100 PRIORITY = 100
@ -15,7 +15,7 @@ CONFIDENCE = 100.0
is_pattern = r"\([^\/]+\)" is_pattern = r"\([^\/]+\)"
def can_handle(pkg, url): def can_handle(pkg):
try: try:
return pkg.metadata._xml_tree.find("upstream").find("watch") \ return pkg.metadata._xml_tree.find("upstream").find("watch") \
is not None is not None
@ -116,7 +116,7 @@ def handle_directory_patterns(base, file_pattern):
for _, path in scan_data] for _, path in scan_data]
def scan(pkg, url): def scan(pkg):
output.einfo("Using watch data") output.einfo("Using watch data")
cp, ver, rev = portage.pkgsplit(pkg.cpv) cp, ver, rev = portage.pkgsplit(pkg.cpv)
@ -126,18 +126,14 @@ def scan(pkg, url):
if not re.search(is_pattern, base): if not re.search(is_pattern, base):
steps = [(base, file_pattern)] steps = [(base, file_pattern)]
res = generic.scan_directory_recursive( res = generic.scan_directory_recursive(
cp, ver, rev, "", steps, url cp, ver, rev, "", steps, base
) )
else: else:
res = [] res = []
for step in handle_directory_patterns(base, file_pattern): for step in handle_directory_patterns(base, file_pattern):
res += generic.scan_directory_recursive( res += generic.scan_directory_recursive(
cp, ver, rev, "", [step], url cp, ver, rev, "", [step], base
) )
results += clean_results(res, versionmangle, urlmangle) results += clean_results(res, versionmangle, urlmangle)
return results return results
def brute_force(pkg, url):
return []

19
pym/euscan/handlers/url/__init__.py Normal file
View File

@ -0,0 +1,19 @@
"""
Url wide handlers for scanning upstream
"""
import pkgutil
handlers = []
# autoimport all modules in this directory and append them to handlers list
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
module = loader.find_module(module_name).load_module(module_name)
handlers.append(module)
# sort handlers by priority
handlers = sorted(
handlers,
key=lambda handler: handler.PRIORITY,
reverse=True
)

27
pym/euscan/handlers/cpan.py → pym/euscan/handlers/url/cpan.py
View File

@ -81,12 +81,22 @@ def cpan_vercmp(cp, a, b):
def scan(pkg, url): def scan(pkg, url):
cp, ver, rev = portage.pkgsplit(pkg.cpv) cp, ver, rev = portage.pkgsplit(pkg.cpv)
pkg = guess_package(cp, url) remote_pkg = guess_package(cp, url)
orig_url = url output.einfo("Using CPAN API: %s", remote_pkg)
url = 'http://search.cpan.org/api/dist/%s' % pkg
output.einfo("Using: " + url) result = scan_remote(pkg, [remote_pkg])
ret = []
for url, pv in result:
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret
def scan_remote(pkg, remote_data):
remote_pkg = remote_data[0]
url = 'http://search.cpan.org/api/dist/%s' % remote_pkg
cp, ver, rev = portage.pkgsplit(pkg.cpv)
try: try:
fp = helpers.urlopen(url) fp = helpers.urlopen(url)
@ -125,13 +135,6 @@ def scan(pkg, url):
version['archive'] version['archive']
) )
if url == orig_url: ret.append((url, pv))
continue
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret return ret
def brute_force(pkg, url):
return []

7
pym/euscan/handlers/generic.py → pym/euscan/handlers/url/generic.py
View File

@ -108,6 +108,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
def scan(pkg, url): def scan(pkg, url):
if CONFIG["scan-dir"]:
for bu in SCANDIR_BLACKLIST_URLS: for bu in SCANDIR_BLACKLIST_URLS:
if re.match(bu, url): if re.match(bu, url):
output.einfo("%s is blacklisted by rule %s" % (url, bu)) output.einfo("%s is blacklisted by rule %s" % (url, bu))
@ -141,10 +142,16 @@ def scan(pkg, url):
steps = helpers.generate_scan_paths(template) steps = helpers.generate_scan_paths(template)
ret = scan_directory_recursive(cp, ver, rev, "", steps, url) ret = scan_directory_recursive(cp, ver, rev, "", steps, url)
if not ret:
ret = brute_force(pkg, url)
return ret return ret
def brute_force(pkg, url): def brute_force(pkg, url):
if CONFIG["brute-force"] == 0:
return []
cp, ver, rev = portage.pkgsplit(pkg.cpv) cp, ver, rev = portage.pkgsplit(pkg.cpv)
url = helpers.parse_mirror(url) url = helpers.parse_mirror(url)

7
pym/euscan/handlers/github.py → pym/euscan/handlers/url/github.py
View File

@ -44,6 +44,7 @@ def scan(pkg, url):
(user, project)) (user, project))
dls = json.load(dlreq) dls = json.load(dlreq)
ret = []
for dl in dls: for dl in dls:
m = fnre.match(dl['name']) m = fnre.match(dl['name'])
@ -51,8 +52,6 @@ def scan(pkg, url):
pv = helpers.gentoo_mangle_version(m.group(1)) pv = helpers.gentoo_mangle_version(m.group(1))
if helpers.version_filtered(cp, ver, pv): if helpers.version_filtered(cp, ver, pv):
continue continue
yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE)
ret.append((dl['html_url'], pv, HANDLER_NAME, CONFIDENCE))
def brute_force(pkg, url): return ret
return []

7
pym/euscan/handlers/kde.py → pym/euscan/handlers/url/kde.py
View File

@ -1,4 +1,4 @@
from euscan.handlers import generic from euscan.handlers.url import generic
PRIORITY = 90 PRIORITY = 90
@ -29,10 +29,7 @@ def scan(pkg, url):
url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/') url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/')
results += generic.scan(pkg.cpv, url) results += generic.scan(pkg.cpv, url)
return clean_results(results) if not results: # if nothing was found go brute forcing
def brute_force(pkg, url):
results = generic.brute_force(pkg.cpv, url) results = generic.brute_force(pkg.cpv, url)
if url.startswith('mirror://kde/unstable/'): if url.startswith('mirror://kde/unstable/'):

4
pym/euscan/handlers/php.py → pym/euscan/handlers/url/php.py
View File

@ -70,7 +70,3 @@ def scan(pkg, url):
ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret return ret
def brute_force(pkg, url):
return []

17
pym/euscan/handlers/pypi.py → pym/euscan/handlers/url/pypi.py
View File

@ -29,6 +29,15 @@ def scan(pkg, url):
package = guess_package(pkg.cpv, url) package = guess_package(pkg.cpv, url)
ret = []
for urls, pv in scan_remote(pkg, [package]):
ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
return ret
def scan_remote(pkg, remote_data):
package = remote_data[0]
output.einfo("Using PyPi XMLRPC: " + package) output.einfo("Using PyPi XMLRPC: " + package)
client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi') client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
@ -42,17 +51,11 @@ def scan(pkg, url):
cp, ver, rev = portage.pkgsplit(pkg.cpv) cp, ver, rev = portage.pkgsplit(pkg.cpv)
ret = [] ret = []
for up_pv in versions: for up_pv in versions:
pv = helpers.gentoo_mangle_version(up_pv) pv = helpers.gentoo_mangle_version(up_pv)
if helpers.version_filtered(cp, ver, pv): if helpers.version_filtered(cp, ver, pv):
continue continue
urls = client.release_urls(package, up_pv) urls = client.release_urls(package, up_pv)
urls = " ".join([infos['url'] for infos in urls]) urls = " ".join([infos['url'] for infos in urls])
ret.append((urls, pv, HANDLER_NAME, CONFIDENCE)) ret.append((urls, pv))
return ret return ret
def brute_force(pkg, url):
return []

26
pym/euscan/handlers/rubygem.py → pym/euscan/handlers/url/rubygems.py
View File

@ -5,7 +5,7 @@ import urllib2
from euscan import helpers, output from euscan import helpers, output
HANDLER_NAME = "rubygem" HANDLER_NAME = "rubygems"
CONFIDENCE = 100.0 CONFIDENCE = 100.0
PRIORITY = 90 PRIORITY = 90
@ -33,14 +33,23 @@ def scan(pkg, url):
'http://guides.rubygems.org/rubygems-org-api/#gemversion' 'http://guides.rubygems.org/rubygems-org-api/#gemversion'
gem = guess_gem(pkg.cpv, url) gem = guess_gem(pkg.cpv, url)
if not gem: if not gem:
output.eerror("Can't guess gem name using %s and %s" % \ output.eerror("Can't guess gem name using %s and %s" % \
(pkg.cpv, url)) (pkg.cpv, url))
return [] return []
url = 'http://rubygems.org/api/v1/versions/%s.json' % gem output.einfo("Using RubyGem API: %s" % gem)
output.einfo("Using: " + url) ret = []
for url, pv in scan_remote(pkg, [gem]):
ret.append(url, pv, HANDLER_NAME, CONFIDENCE)
return ret
def scan_remote(pkg, remote_data):
gem = remote_data[0]
url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
try: try:
fp = helpers.urlopen(url) fp = helpers.urlopen(url)
@ -55,23 +64,14 @@ def scan(pkg, url):
data = fp.read() data = fp.read()
versions = json.loads(data) versions = json.loads(data)
if not versions:
return []
cp, ver, rev = portage.pkgsplit(pkg.cpv) cp, ver, rev = portage.pkgsplit(pkg.cpv)
ret = [] ret = []
for version in versions: for version in versions:
up_pv = version['number'] up_pv = version['number']
pv = helpers.gentoo_mangle_version(up_pv) pv = helpers.gentoo_mangle_version(up_pv)
if helpers.version_filtered(cp, ver, pv): if helpers.version_filtered(cp, ver, pv):
continue continue
url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv) url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) ret.append((url, pv))
return ret return ret
def brute_force(pkg, url):
return []

72
pym/euscan/scan.py
View File

@ -44,60 +44,6 @@ def filter_versions(cp, versions):
] ]
def scan_upstream_urls(pkg, urls, on_progress):
versions = []
if on_progress:
progress_available = 70
num_urls = sum([len(urls[fn]) for fn in urls])
if num_urls > 0:
progress_increment = progress_available / num_urls
else:
progress_increment = 0
for filename in urls:
for url in urls[filename]:
if on_progress and progress_available > 0:
on_progress(increment=progress_increment)
progress_available -= progress_increment
if not CONFIG['quiet'] and not CONFIG['format']:
pp.uprint()
output.einfo("SRC_URI is '%s'" % url)
if '://' not in url:
output.einfo("Invalid url '%s'" % url)
continue
# Try normal scan
if CONFIG["scan-dir"]:
try:
versions.extend(handlers.scan(pkg, url))
except Exception as e:
output.ewarn("Handler failed: [%s] %s"
% (e.__class__.__name__, e.message))
if versions and CONFIG['oneshot']:
break
# Brute Force
if CONFIG["brute-force"] > 0:
versions.extend(handlers.brute_force(pkg, url))
if versions and CONFIG['oneshot']:
break
cp, ver, rev = portage.pkgsplit(pkg.cpv)
result = filter_versions(cp, versions)
if on_progress and progress_available > 0:
on_progress(increment=progress_available)
return result
# gentoolkit stores PORTDB, so even if we modify it to add an overlay # gentoolkit stores PORTDB, so even if we modify it to add an overlay
# it will still use the old dbapi # it will still use the old dbapi
def reload_gentoolkit(): def reload_gentoolkit():
@ -120,7 +66,6 @@ def scan_upstream(query, on_progress=None):
""" """
Scans the upstream searching new versions for the given query Scans the upstream searching new versions for the given query
""" """
matches = [] matches = []
if query.endswith(".ebuild"): if query.endswith(".ebuild"):
@ -188,10 +133,6 @@ def scan_upstream(query, on_progress=None):
output.metadata("description", pkg.environment("DESCRIPTION")) output.metadata("description", pkg.environment("DESCRIPTION"))
cpv = pkg.cpv cpv = pkg.cpv
_, _, ver, _ = portage.catpkgsplit(cpv)
is_current_version_stable = is_version_stable(ver)
metadata = { metadata = {
"EAPI": portage.settings["EAPI"], "EAPI": portage.settings["EAPI"],
"SRC_URI": pkg.environment("SRC_URI", False), "SRC_URI": pkg.environment("SRC_URI", False),
@ -212,15 +153,20 @@ def scan_upstream(query, on_progress=None):
else: else:
urls = alist urls = alist
# output scan time for formatted output versions = handlers.scan(pkg, urls, on_progress)
scan_time = (datetime.now() - start_time).total_seconds()
output.metadata("scan_time", scan_time, show=False)
result = scan_upstream_urls(pkg, urls, on_progress) cp, ver, rev = portage.pkgsplit(pkg.cpv)
result = filter_versions(cp, versions)
if on_progress: if on_progress:
on_progress(increment=10) on_progress(increment=10)
# output scan time for formatted output
scan_time = (datetime.now() - start_time).total_seconds()
output.metadata("scan_time", scan_time, show=False)
is_current_version_stable = is_version_stable(ver)
if len(result) > 0: if len(result) > 0:
if not (CONFIG['format'] or CONFIG['quiet']): if not (CONFIG['format'] or CONFIG['quiet']):
print("\n", file=sys.stderr) print("\n", file=sys.stderr)