src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'fox/master'

Browse Source 
Conflicts:
	euscanwww/djeuscan/helpers.py

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>
This commit is contained in:
Corentin Chary 2012-07-26 21:17:10 +02:00
commit 29b1ad557a
26 changed files with 992 additions and 376 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bin/euscan
View File

@ -129,9 +129,9 @@ def print_usage(_error=None, help=None):
" - Ignore non-stable versions only if current\n" +
" " * 38 + "version is stable", file=out)
print(yellow(" --mirror") +
" - use mirror:// URIs", file=out)
" - use mirror:// URIs", file=out)
print(yellow(" --ebuild-uri") +
" - use ebuild variables in URIs", file=out)
" - use ebuild variables in URIs", file=out)
print(file=out)
if _error in ('packages',) or help:

32
bin/euscan_patch_metadata
View File

@ -9,7 +9,7 @@ import tarfile
import gzip
import logging
import shutil
import subprocess
from difflib import unified_diff
from portage.exception import AmbiguousPackageName
from gentoolkit.query import Query
@ -117,7 +117,7 @@ def get_deb_url(name):
content = opened.read()
for link in BeautifulSoup(content, parseOnlyThese=SoupStrainer("a")):
if re.match("[^\s]+\.debian\.tar\.gz", link.text):
if re.match("[^\s]+\.debian\.tar\.(?:gz|bz2)", link.text):
deb_url = link["href"]
deb_type = "source"
break
@ -133,9 +133,11 @@ def get_deb_url(name):
return deb_url, deb_type
def patch_metadata(metadata_path, watch_data, diff=False):
def patch_metadata(package, watch_data, diff=False):
logger.info(" Patching metadata file")
metadata_path = package.metadata.metadata_path
with open(metadata_path) as fp:
original = fp.read()
rindent, indent = guess_indent_values(original)
@ -203,14 +205,17 @@ def patch_metadata(metadata_path, watch_data, diff=False):
if not diff:
return data
else:
_, data_path = mkstemp()
with open(data_path, "w") as f:
f.write(data)
sub = subprocess.Popen(["diff", metadata_path, data_path],
stdout=subprocess.PIPE)
content = sub.stdout.read()
os.unlink(data_path)
return content
# Generate clean a/category/package/metadata.xml path
n = metadata_path.find(package.category)
if n != -1:
metadata_path = metadata_path[n:]
res = unified_diff(
original.splitlines(True),
data.splitlines(True),
fromfile=os.path.join('a/', metadata_path),
tofile=os.path.join('b/', metadata_path),
)
return "".join([x for x in res])
def process_package(query, diff=False):
@ -236,12 +241,11 @@ def process_package(query, diff=False):
if '9999' in package.version and len(matches) > 0:
package = matches.pop()
metadata_path = package.metadata.metadata_path
watch_data = get_watch_data(package)
if watch_data is None:
logger.error(" No watch file found")
else:
return patch_metadata(metadata_path, watch_data, diff=diff)
return patch_metadata(package, watch_data, diff=diff)
def main():
@ -261,7 +265,7 @@ def main():
logger.info("Processing %s..." % package)
result = process_package(package, opts.diff)
if result:
print result
sys.stdout.write(result)
if __name__ == "__main__":
main()

9
euscanwww/djeuscan/admin.py
View File

@ -1,7 +1,6 @@
from djeuscan.models import Package, Version, VersionLog, EuscanResult, \
Log, WorldLog, CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer, \
RefreshPackageQuery, HerdAssociation, CategoryAssociation, \
MaintainerAssociation, PackageAssociation, ProblemReport
RefreshPackageQuery, Category, Overlay, ProblemReport
from django.contrib import admin
@ -54,9 +53,7 @@ admin.site.register(HerdLog)
admin.site.register(MaintainerLog)
admin.site.register(RefreshPackageQuery)
admin.site.register(HerdAssociation)
admin.site.register(CategoryAssociation)
admin.site.register(MaintainerAssociation)
admin.site.register(PackageAssociation)
admin.site.register(Category)
admin.site.register(Overlay)
admin.site.register(ProblemReport, ProblemReportAdmin)

30
euscanwww/djeuscan/helpers.py
View File

@ -76,33 +76,33 @@ class catch_and_return(object):
return wrapper
def get_account_categories(user):
from djeuscan.models import Package, CategoryAssociation
def get_profile(user):
from djeuscan.models import UserProfile
try:
return user.get_profile()
except UserProfile.DoesNotExist:
UserProfile.objects.create(user=user)
return user.get_profile()
def get_account_categories(user):
from djeuscan.models import Package
# TODO: This is quite ugly
category_names = [obj.category for obj in
CategoryAssociation.objects.filter(user=user)]
category_names = [obj.name for obj in get_profile(user).categories.all()]
return [c for c in Package.objects.categories()
if c["category"] in category_names]
def get_account_herds(user):
from djeuscan.models import Package, HerdAssociation
from djeuscan.models import Package
ids = [obj.herd.pk for obj in
HerdAssociation.objects.filter(user=user)]
ids = [herd.pk for herd in get_profile(user).herds.all()]
return Package.objects.herds(ids=ids)
def get_account_maintainers(user):
from djeuscan.models import Package, MaintainerAssociation
from djeuscan.models import Package
ids = [obj.maintainer.pk for obj in
MaintainerAssociation.objects.filter(user=user)]
ids = [obj.pk for obj in get_profile(user).maintainers.all()]
return Package.objects.maintainers(ids=ids)
def get_account_packages(user):
from djeuscan.models import PackageAssociation
return [obj.package for obj in
PackageAssociation.objects.filter(user=user)]

355
euscanwww/djeuscan/migrations/0016_auto__del_maintainerassociation__del_unique_maintainerassociation_user.py Normal file
View File

@ -0,0 +1,355 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Removing unique constraint on 'CategoryAssociation', fields ['user', 'category']
db.delete_unique('djeuscan_categoryassociation', ['user_id', 'category'])
# Removing unique constraint on 'OverlayAssociation', fields ['user', 'overlay']
db.delete_unique('djeuscan_overlayassociation', ['user_id', 'overlay'])
# Removing unique constraint on 'HerdAssociation', fields ['user', 'herd']
db.delete_unique('djeuscan_herdassociation', ['user_id', 'herd_id'])
# Removing unique constraint on 'PackageAssociation', fields ['user', 'package']
db.delete_unique('djeuscan_packageassociation', ['user_id', 'package_id'])
# Removing unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer']
db.delete_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id'])
# Deleting model 'MaintainerAssociation'
db.delete_table('djeuscan_maintainerassociation')
# Deleting model 'PackageAssociation'
db.delete_table('djeuscan_packageassociation')
# Deleting model 'HerdAssociation'
db.delete_table('djeuscan_herdassociation')
# Deleting model 'OverlayAssociation'
db.delete_table('djeuscan_overlayassociation')
# Deleting model 'CategoryAssociation'
db.delete_table('djeuscan_categoryassociation')
# Adding model 'UserProfile'
db.create_table('djeuscan_userprofile', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('user', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['auth.User'], unique=True)),
))
db.send_create_signal('djeuscan', ['UserProfile'])
# Adding M2M table for field herds on 'UserProfile'
db.create_table('djeuscan_userprofile_herds', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)),
('herd', models.ForeignKey(orm['djeuscan.herd'], null=False))
))
db.create_unique('djeuscan_userprofile_herds', ['userprofile_id', 'herd_id'])
# Adding M2M table for field maintainers on 'UserProfile'
db.create_table('djeuscan_userprofile_maintainers', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)),
('maintainer', models.ForeignKey(orm['djeuscan.maintainer'], null=False))
))
db.create_unique('djeuscan_userprofile_maintainers', ['userprofile_id', 'maintainer_id'])
# Adding M2M table for field packages on 'UserProfile'
db.create_table('djeuscan_userprofile_packages', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)),
('package', models.ForeignKey(orm['djeuscan.package'], null=False))
))
db.create_unique('djeuscan_userprofile_packages', ['userprofile_id', 'package_id'])
# Adding M2M table for field categories on 'UserProfile'
db.create_table('djeuscan_userprofile_categories', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)),
('category', models.ForeignKey(orm['djeuscan.category'], null=False))
))
db.create_unique('djeuscan_userprofile_categories', ['userprofile_id', 'category_id'])
# Adding M2M table for field overlays on 'UserProfile'
db.create_table('djeuscan_userprofile_overlays', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)),
('overlay', models.ForeignKey(orm['djeuscan.overlay'], null=False))
))
db.create_unique('djeuscan_userprofile_overlays', ['userprofile_id', 'overlay_id'])
# Adding model 'Category'
db.create_table('djeuscan_category', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('name', self.gf('django.db.models.fields.CharField')(max_length=128)),
))
db.send_create_signal('djeuscan', ['Category'])
# Adding model 'Overlay'
db.create_table('djeuscan_overlay', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('name', self.gf('django.db.models.fields.CharField')(max_length=128)),
))
db.send_create_signal('djeuscan', ['Overlay'])
def backwards(self, orm):
# Adding model 'MaintainerAssociation'
db.create_table('djeuscan_maintainerassociation', (
('maintainer', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Maintainer'])),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
))
db.send_create_signal('djeuscan', ['MaintainerAssociation'])
# Adding unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer']
db.create_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id'])
# Adding model 'PackageAssociation'
db.create_table('djeuscan_packageassociation', (
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('package', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Package'])),
))
db.send_create_signal('djeuscan', ['PackageAssociation'])
# Adding unique constraint on 'PackageAssociation', fields ['user', 'package']
db.create_unique('djeuscan_packageassociation', ['user_id', 'package_id'])
# Adding model 'HerdAssociation'
db.create_table('djeuscan_herdassociation', (
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('herd', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Herd'])),
))
db.send_create_signal('djeuscan', ['HerdAssociation'])
# Adding unique constraint on 'HerdAssociation', fields ['user', 'herd']
db.create_unique('djeuscan_herdassociation', ['user_id', 'herd_id'])
# Adding model 'OverlayAssociation'
db.create_table('djeuscan_overlayassociation', (
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('overlay', self.gf('django.db.models.fields.CharField')(max_length=128)),
))
db.send_create_signal('djeuscan', ['OverlayAssociation'])
# Adding unique constraint on 'OverlayAssociation', fields ['user', 'overlay']
db.create_unique('djeuscan_overlayassociation', ['user_id', 'overlay'])
# Adding model 'CategoryAssociation'
db.create_table('djeuscan_categoryassociation', (
('category', self.gf('django.db.models.fields.CharField')(max_length=128)),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
))
db.send_create_signal('djeuscan', ['CategoryAssociation'])
# Adding unique constraint on 'CategoryAssociation', fields ['user', 'category']
db.create_unique('djeuscan_categoryassociation', ['user_id', 'category'])
# Deleting model 'UserProfile'
db.delete_table('djeuscan_userprofile')
# Removing M2M table for field herds on 'UserProfile'
db.delete_table('djeuscan_userprofile_herds')
# Removing M2M table for field maintainers on 'UserProfile'
db.delete_table('djeuscan_userprofile_maintainers')
# Removing M2M table for field packages on 'UserProfile'
db.delete_table('djeuscan_userprofile_packages')
# Removing M2M table for field categories on 'UserProfile'
db.delete_table('djeuscan_userprofile_categories')
# Removing M2M table for field overlays on 'UserProfile'
db.delete_table('djeuscan_userprofile_overlays')
# Deleting model 'Category'
db.delete_table('djeuscan_category')
# Deleting model 'Overlay'
db.delete_table('djeuscan_overlay')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'djeuscan.category': {
'Meta': {'object_name': 'Category'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.categorylog': {
'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.euscanresult': {
'Meta': {'object_name': 'EuscanResult'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'})
},
'djeuscan.herd': {
'Meta': {'object_name': 'Herd'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'})
},
'djeuscan.herdlog': {
'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.log': {
'Meta': {'object_name': 'Log'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'djeuscan.maintainer': {
'Meta': {'object_name': 'Maintainer'},
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.maintainerlog': {
'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
},
'djeuscan.overlay': {
'Meta': {'object_name': 'Overlay'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.package': {
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.problemreport': {
'Meta': {'object_name': 'ProblemReport'},
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'message': ('django.db.models.fields.TextField', [], {}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'subject': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Version']", 'null': 'True', 'blank': 'True'})
},
'djeuscan.refreshpackagequery': {
'Meta': {'object_name': 'RefreshPackageQuery'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'symmetrical': 'False'})
},
'djeuscan.userprofile': {
'Meta': {'object_name': 'UserProfile'},
'categories': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Category']", 'symmetrical': 'False'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'}),
'overlays': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Overlay']", 'symmetrical': 'False'}),
'packages': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Package']", 'symmetrical': 'False'}),
'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'})
},
'djeuscan.version': {
'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'ebuild_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'metadata_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
},
'djeuscan.versionlog': {
'Meta': {'object_name': 'VersionLog'},
'action': ('django.db.models.fields.IntegerField', [], {}),
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
},
'djeuscan.worldlog': {
'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
}
}
complete_apps = ['djeuscan']

197
euscanwww/djeuscan/migrations/0017_auto__add_unique_overlay_name__add_unique_category_name.py Normal file
View File

@ -0,0 +1,197 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding unique constraint on 'Overlay', fields ['name']
db.create_unique('djeuscan_overlay', ['name'])
# Adding unique constraint on 'Category', fields ['name']
db.create_unique('djeuscan_category', ['name'])
def backwards(self, orm):
# Removing unique constraint on 'Category', fields ['name']
db.delete_unique('djeuscan_category', ['name'])
# Removing unique constraint on 'Overlay', fields ['name']
db.delete_unique('djeuscan_overlay', ['name'])
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'djeuscan.category': {
'Meta': {'object_name': 'Category'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'})
},
'djeuscan.categorylog': {
'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.euscanresult': {
'Meta': {'object_name': 'EuscanResult'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'})
},
'djeuscan.herd': {
'Meta': {'object_name': 'Herd'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'})
},
'djeuscan.herdlog': {
'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']},
'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}),
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
},
'djeuscan.log': {
'Meta': {'object_name': 'Log'},
'datetime': ('django.db.models.fields.DateTimeField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'djeuscan.maintainer': {
'Meta': {'object_name': 'Maintainer'},
'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.maintainerlog': {
'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}),
'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"})
},
'djeuscan.overlay': {
'Meta': {'object_name': 'Overlay'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'})
},
'djeuscan.package': {
'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'},
'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}),
'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}),
'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '128'})
},
'djeuscan.problemreport': {
'Meta': {'object_name': 'ProblemReport'},
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'message': ('django.db.models.fields.TextField', [], {}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'subject': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Version']", 'null': 'True', 'blank': 'True'})
},
'djeuscan.refreshpackagequery': {
'Meta': {'object_name': 'RefreshPackageQuery'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'symmetrical': 'False'})
},
'djeuscan.userprofile': {
'Meta': {'object_name': 'UserProfile'},
'categories': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Category']", 'symmetrical': 'False'}),
'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'}),
'overlays': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Overlay']", 'symmetrical': 'False'}),
'packages': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Package']", 'symmetrical': 'False'}),
'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'})
},
'djeuscan.version': {
'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'},
'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'ebuild_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'metadata_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
},
'djeuscan.versionlog': {
'Meta': {'object_name': 'VersionLog'},
'action': ('django.db.models.fields.IntegerField', [], {}),
'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}),
'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}),
'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}),
'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'})
},
'djeuscan.worldlog': {
'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']},
'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'})
}
}
complete_apps = ['djeuscan']

80
euscanwww/djeuscan/models.py
View File

@ -231,6 +231,31 @@ class EuscanResult(models.Model):
)
class Category(models.Model):
name = models.CharField(max_length=128, validators=[validate_category],
unique=True)
def __unicode__(self):
return self.name
class Overlay(models.Model):
name = models.CharField(max_length=128, validators=[validate_name],
unique=True)
def __unicode__(self):
return self.name
class UserProfile(models.Model):
user = models.OneToOneField(User)
herds = models.ManyToManyField(Herd)
maintainers = models.ManyToManyField(Maintainer)
packages = models.ManyToManyField(Package)
categories = models.ManyToManyField(Category)
overlays = models.ManyToManyField(Overlay)
class Log(models.Model):
"""
Model used for keeping data for charts
@ -303,61 +328,6 @@ class RefreshPackageQuery(models.Model):
return u'[%d] %s' % (self.priority, self.package)
class HerdAssociation(models.Model):
user = models.ForeignKey(User)
herd = models.ForeignKey(Herd)
class Meta:
unique_together = ['user', 'herd']
def __unicode__(self):
return u'[%s] %s' % (self.user, self.herd)
class MaintainerAssociation(models.Model):
user = models.ForeignKey(User)
maintainer = models.ForeignKey(Maintainer)
class Meta:
unique_together = ['user', 'maintainer']
def __unicode__(self):
return u'[%s] %s' % (self.user, self.maintainer)
class PackageAssociation(models.Model):
user = models.ForeignKey(User)
package = models.ForeignKey(Package)
class Meta:
unique_together = ['user', 'package']
def __unicode__(self):
return u'[%s] %s' % (self.user, self.package)
class CategoryAssociation(models.Model):
user = models.ForeignKey(User)
category = models.CharField(max_length=128, validators=[validate_category])
class Meta:
unique_together = ['user', 'category']
def __unicode__(self):
return u'[%s] %s' % (self.user, self.category)
class OverlayAssociation(models.Model):
user = models.ForeignKey(User)
overlay = models.CharField(max_length=128, validators=[validate_name])
class Meta:
unique_together = ['user', 'overlay']
def __unicode__(self):
return u'[%s] %s' % (self.user, self.category)
class ProblemReport(models.Model):
package = models.ForeignKey(Package)
version = models.ForeignKey(Version, null=True, blank=True)

23
euscanwww/djeuscan/processing/misc/update_counters.py
View File

@ -1,8 +1,8 @@
from django.db.transaction import commit_on_success
from django.utils import timezone
from djeuscan.models import Package, Herd, Maintainer, Version
from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog
from djeuscan.models import Package, Herd, Maintainer, Version, HerdLog, \
MaintainerLog, CategoryLog, WorldLog, Category, Overlay
from djeuscan import charts
from djeuscan.processing import FakeLogger
@ -48,6 +48,19 @@ def update_counters(fast=False, nolog=False, logger=None):
wlog = None
# Populate Category and Overlay
for cat in Package.objects.values('category').distinct():
obj, created = Category.objects.get_or_create(name=cat["category"])
if created:
logger.info("+ [c] %s", cat["category"])
for overlay in Version.objects.values('overlay').distinct():
if not overlay["overlay"]:
continue
obj, created = Overlay.objects.get_or_create(name=overlay["overlay"])
if created:
logger.info("+ [o] %s", overlay["overlay"])
if not nolog:
wlog = WorldLog()
wlog.datetime = now
@ -157,17 +170,17 @@ def update_counters(fast=False, nolog=False, logger=None):
return
for clog in categories.values():
logger.info('+ [cl] %s' % clog)
logger.info('+ [cl] %s', clog)
charts.rrd_update('category-%s' % clog.category, now, clog)
clog.save()
for hlog in herds.values():
logger.info('+ [hl] %s' % hlog)
logger.info('+ [hl] %s', hlog)
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
hlog.save()
for mlog in maintainers.values():
logger.info('+ [ml] %s' % mlog)
logger.info('+ [ml] %s', mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save()

39
euscanwww/djeuscan/tests/euscan_factory.py
View File

@ -4,10 +4,12 @@ from datetime import datetime
from collections import defaultdict
from django.contrib.auth.models import User
from django.db import IntegrityError
import factory
from djeuscan.models import Herd, Maintainer, Package, Version, EuscanResult
from djeuscan.models import Herd, Maintainer, Package, Version, EuscanResult, \
Category, Overlay
class UserFactory(factory.Factory):
@ -36,6 +38,18 @@ class MaintainerFactory(factory.Factory):
email = factory.LazyAttribute(lambda a: "%s@example.com" % a.name)
class CategoryFactory(factory.Factory):
FACTORY_FOR = Category
name = factory.LazyAttribute(lambda a: random_string())
class OverlayFactory(factory.Factory):
FACTORY_FOR = Overlay
name = factory.LazyAttribute(lambda a: random_string())
class PackageFactory(factory.Factory):
FACTORY_FOR = Package
@ -46,6 +60,16 @@ class PackageFactory(factory.Factory):
description = "This is a test package"
homepage = "http://testpackage.com"
@classmethod
def _prepare(cls, create, **kwargs):
package = super(PackageFactory, cls)._prepare(create, **kwargs)
category = kwargs.pop('category', None)
if create:
CategoryFactory.create(name=category)
else:
CategoryFactory.build(name=category)
return package
class VersionFactory(factory.Factory):
FACTORY_FOR = Version
@ -59,6 +83,19 @@ class VersionFactory(factory.Factory):
urls = "http://packageurl.com"
alive = True
@classmethod
def _prepare(cls, create, **kwargs):
version = super(VersionFactory, cls)._prepare(create, **kwargs)
overlay = kwargs.pop('overlay', None)
try:
if create:
OverlayFactory.create(name=overlay)
else:
OverlayFactory.build(name=overlay)
except IntegrityError:
pass
return version
class EuscanResultFactory(factory.Factory):
FACTORY_FOR = EuscanResult

42
euscanwww/djeuscan/tests/test_views.py
View File

@ -5,13 +5,12 @@ try:
except ImportError:
from bs4 import BeautifulSoup
from djeuscan.helpers import get_profile
from djeuscan.tests import SystemTestCase
from djeuscan.tests.euscan_factory import PackageFactory, setup_maintainers, \
setup_herds, setup_categories, setup_overlays
from djeuscan.models import PackageAssociation, CategoryAssociation, \
HerdAssociation, MaintainerAssociation
class PagesTest(SystemTestCase):
"""
@ -50,11 +49,10 @@ class PackageTests(SystemTestCase):
self.assertEqual(response.status_code, 200)
def test_favourite(self):
self.assertEqual(PackageAssociation.objects.count(), 0)
response = self.get("package", category=self.package.category,
package=self.package.name)
self.assertEqual(response.status_code, 200)
self.assertNotIn("Watch", response.content)
with self.login():
@ -62,11 +60,14 @@ class PackageTests(SystemTestCase):
package=self.package.name)
self.assertEqual(response.status_code, 200)
user = response.context["user"]
self.assertEquals(get_profile(user).categories.count(), 0)
self.assertIn("Watch", response.content)
self.post("favourite_package", category=self.package.category,
package=self.package.name)
self.assertEqual(PackageAssociation.objects.count(), 1)
self.assertEquals(get_profile(user).packages.count(), 1)
response = self.get("accounts_packages")
self.assertEqual(response.status_code, 200)
@ -75,7 +76,7 @@ class PackageTests(SystemTestCase):
self.post("unfavourite_package", category=self.package.category,
package=self.package.name)
self.assertEqual(PackageAssociation.objects.count(), 0)
self.assertEquals(get_profile(user).packages.count(), 0)
class SectionTests(SystemTestCase):
@ -118,9 +119,9 @@ class CategoriesTests(SectionTests):
def test_favourite(self):
category = self.categories[0]
self.assertEqual(CategoryAssociation.objects.count(), 0)
response = self.get("category", category=category)
self.assertEqual(response.status_code, 200)
self.assertNotIn("Watch", response.content)
@ -128,10 +129,13 @@ class CategoriesTests(SectionTests):
response = self.get("category", category=category)
self.assertEqual(response.status_code, 200)
user = response.context["user"]
self.assertEquals(get_profile(user).categories.count(), 0)
self.assertIn("Watch", response.content)
self.post("favourite_category", category=category)
self.assertEqual(CategoryAssociation.objects.count(), 1)
self.assertEquals(get_profile(user).categories.count(), 1)
response = self.get("accounts_categories")
self.assertEqual(response.status_code, 200)
@ -139,7 +143,8 @@ class CategoriesTests(SectionTests):
self._check_table(response, [category])
self.post("unfavourite_category", category=category)
self.assertEqual(CategoryAssociation.objects.count(), 0)
self.assertEquals(get_profile(user).categories.count(), 0)
class HerdsTests(SectionTests):
@ -167,7 +172,6 @@ class HerdsTests(SectionTests):
def test_favourite(self):
herd = self.herds[0]
self.assertEqual(HerdAssociation.objects.count(), 0)
response = self.get("herd", herd=herd.herd)
self.assertEqual(response.status_code, 200)
@ -177,10 +181,13 @@ class HerdsTests(SectionTests):
response = self.get("herd", herd=herd.herd)
self.assertEqual(response.status_code, 200)
user = response.context["user"]
self.assertEquals(get_profile(user).herds.count(), 0)
self.assertIn("Watch", response.content)
self.post("favourite_herd", herd=herd.herd)
self.assertEqual(HerdAssociation.objects.count(), 1)
self.assertEquals(get_profile(user).herds.count(), 1)
response = self.get("accounts_herds")
self.assertEqual(response.status_code, 200)
@ -188,7 +195,7 @@ class HerdsTests(SectionTests):
self._check_table(response, [herd], attr="herd")
self.post("unfavourite_herd", herd=herd.herd)
self.assertEqual(HerdAssociation.objects.count(), 0)
self.assertEquals(get_profile(user).herds.count(), 0)
class MaintainersTests(SectionTests):
@ -216,7 +223,6 @@ class MaintainersTests(SectionTests):
def test_favourite(self):
maintainer = self.maintainers[0]
self.assertEqual(MaintainerAssociation.objects.count(), 0)
response = self.get("maintainer", maintainer_id=maintainer.pk)
self.assertEqual(response.status_code, 200)
@ -226,10 +232,13 @@ class MaintainersTests(SectionTests):
response = self.get("maintainer", maintainer_id=maintainer.pk)
self.assertEqual(response.status_code, 200)
user = response.context["user"]
self.assertEquals(get_profile(user).maintainers.count(), 0)
self.assertIn("Watch", response.content)
self.post("favourite_maintainer", maintainer_id=maintainer.pk)
self.assertEqual(MaintainerAssociation.objects.count(), 1)
self.assertEquals(get_profile(user).maintainers.count(), 1)
response = self.get("accounts_maintainers")
self.assertEqual(response.status_code, 200)
@ -237,7 +246,8 @@ class MaintainersTests(SectionTests):
self._check_table(response, [maintainer], attr="name")
self.post("unfavourite_maintainer", maintainer_id=maintainer.pk)
self.assertEqual(MaintainerAssociation.objects.count(), 0)
self.assertEquals(get_profile(user).maintainers.count(), 0)
class OverlayTests(SectionTests):

133
euscanwww/djeuscan/views.py
View File

@ -8,12 +8,10 @@ from django.shortcuts import get_object_or_404
from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_POST
from djeuscan.helpers import version_key, packages_from_names, \
get_account_categories, get_account_herds, get_account_maintainers, \
get_account_packages
from djeuscan.helpers import version_key, packages_from_names, get_profile, \
get_account_categories, get_account_herds, get_account_maintainers
from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \
VersionLog, RefreshPackageQuery, HerdAssociation, MaintainerAssociation, \
CategoryAssociation, PackageAssociation, OverlayAssociation, ProblemReport
VersionLog, RefreshPackageQuery, ProblemReport, Category, Overlay
from djeuscan.forms import WorldForm, PackagesForm, ProblemReportForm
from djeuscan.tasks import admin_tasks
from djeuscan import charts
@ -68,12 +66,8 @@ def category(request, category):
favourited = False
if request.user.is_authenticated():
try:
CategoryAssociation.objects.get(user=request.user,
category=category)
except CategoryAssociation.DoesNotExist:
pass
else:
if Category.objects.get(name=category) in \
get_profile(request.user).categories.all():
favourited = True
return {'category': category, 'packages': packages, 'last_scan': last_scan,
@ -104,11 +98,7 @@ def herd(request, herd):
favourited = False
if request.user.is_authenticated():
try:
HerdAssociation.objects.get(user=request.user, herd=herd)
except HerdAssociation.DoesNotExist:
pass
else:
if herd in get_profile(request.user).herds.all():
favourited = True
return {'herd': herd, 'packages': packages, "last_scan": last_scan,
@ -140,12 +130,7 @@ def maintainer(request, maintainer_id):
favourited = False
if request.user.is_authenticated():
try:
MaintainerAssociation.objects.get(user=request.user,
maintainer=maintainer)
except MaintainerAssociation.DoesNotExist:
pass
else:
if maintainer in get_profile(request.user).maintainers.all():
favourited = True
return {'maintainer': maintainer, 'packages': packages,
@ -177,16 +162,14 @@ def overlay(request, overlay):
favourited = False
if request.user.is_authenticated():
try:
OverlayAssociation.objects.get(user=request.user, overlay=overlay)
except OverlayAssociation.DoesNotExist:
pass
else:
if Overlay.objects.get(name=overlay) in \
get_profile(request.user).overlays.all():
favourited = True
return {'overlay': overlay, 'packages': packages, 'last_scan': last_scan,
'favourited': favourited}
@render_to('euscan/package.html')
def package(request, category, package):
package = get_object_or_404(Package, category=category, name=package)
@ -208,11 +191,7 @@ def package(request, category, package):
favourited = False
if request.user.is_authenticated():
try:
PackageAssociation.objects.get(user=request.user, package=package)
except PackageAssociation.DoesNotExist:
pass
else:
if package in get_profile(request.user).packages.all():
favourited = True
try:
@ -225,14 +204,15 @@ def package(request, category, package):
'package': package,
'packaged': packaged,
'upstream': upstream,
'log': log.messages(),
'vlog': vlog,
'msg' : log.messages() if log else "",
'log': log,
'msg': log.messages() if log else "",
'last_scan': last_scan,
'favourited': favourited,
'refreshed': refreshed,
}
@login_required
@render_to('euscan/problem.html')
def problem(request, category, package):
@ -265,8 +245,9 @@ def problem(request, category, package):
'package': package,
'packaged': packaged,
'upstream': upstream,
'msg' : log.messages() if log else "",
}
'msg': log.messages() if log else "",
}
@render_to('euscan/world.html')
def world(request):
@ -276,7 +257,7 @@ def world(request):
return {
'world_form': world_form,
'packages_form': packages_form
}
}
@render_to('euscan/world_scan.html')
@ -394,9 +375,10 @@ def refresh_package(request, category, package):
@login_required
@render_to('euscan/accounts/index.html')
def accounts_index(request):
user = request.user
upstream_k = lambda c: c["n_versions"] - c["n_packaged"]
categories = sorted(get_account_categories(request.user),
categories = sorted(get_account_categories(user),
key=upstream_k, reverse=True)
c_upstream = sum([c["n_versions"] - c["n_packaged"] for c in categories])
herds = sorted(get_account_herds(request.user),
@ -405,7 +387,7 @@ def accounts_index(request):
maintainers = sorted(get_account_maintainers(request.user),
key=upstream_k, reverse=True)
m_upstream = sum([c["n_versions"] - c["n_packaged"] for c in maintainers])
packages = sorted(get_account_packages(request.user),
packages = sorted(get_profile(user).packages.all(),
key=lambda p: p.n_versions - p.n_packaged, reverse=True)
p_upstream = sum([c.n_versions - c.n_packaged for c in packages])
return {
@ -437,14 +419,13 @@ def accounts_maintainers(request):
@login_required
@render_to('euscan/accounts/packages.html')
def accounts_packages(request):
return {"packages": get_account_packages(request.user)}
return {"packages": get_profile(request.user).packages.all()}
@login_required
@render_to('euscan/accounts/overlays.html')
def accounts_overlays(request):
overlays = [obj.overlay for obj in
OverlayAssociation.objects.filter(user=request.user)]
overlays = [obj.name for obj in get_profile(request.user).overlays.all()]
return {"overlays": overlays}
@ -453,10 +434,8 @@ def accounts_overlays(request):
@ajax_request
def favourite_package(request, category, package):
obj = get_object_or_404(Package, category=category, name=package)
_, created = PackageAssociation.objects.get_or_create(
user=request.user, package=obj
)
return {"success": created}
get_profile(request.user).packages.add(obj)
return {"success": True}
@login_required
@ -464,10 +443,7 @@ def favourite_package(request, category, package):
@ajax_request
def unfavourite_package(request, category, package):
package = get_object_or_404(Package, category=category, name=package)
obj = get_object_or_404(
PackageAssociation, package=package, user=request.user
)
obj.delete()
get_profile(request.user).packages.remove(package)
return {"success": True}
@ -476,10 +452,8 @@ def unfavourite_package(request, category, package):
@ajax_request
def favourite_herd(request, herd):
obj = get_object_or_404(Herd, herd=herd)
_, created = HerdAssociation.objects.get_or_create(
user=request.user, herd=obj
)
return {"success": created}
get_profile(request.user).herds.add(obj)
return {"success": True}
@login_required
@ -487,10 +461,7 @@ def favourite_herd(request, herd):
@ajax_request
def unfavourite_herd(request, herd):
herd = get_object_or_404(Herd, herd=herd)
obj = get_object_or_404(
HerdAssociation, herd=herd, user=request.user
)
obj.delete()
get_profile(request.user).herds.remove(herd)
return {"success": True}
@ -499,10 +470,8 @@ def unfavourite_herd(request, herd):
@ajax_request
def favourite_maintainer(request, maintainer_id):
obj = get_object_or_404(Maintainer, pk=maintainer_id)
_, created = MaintainerAssociation.objects.get_or_create(
user=request.user, maintainer=obj
)
return {"success": created}
get_profile(request.user).maintainers.add(obj)
return {"success": True}
@login_required
@ -510,10 +479,7 @@ def favourite_maintainer(request, maintainer_id):
@ajax_request
def unfavourite_maintainer(request, maintainer_id):
maintainer = get_object_or_404(Maintainer, pk=maintainer_id)
obj = get_object_or_404(
MaintainerAssociation, maintainer=maintainer, user=request.user
)
obj.delete()
get_profile(request.user).maintainers.remove(maintainer)
return {"success": True}
@ -521,25 +487,17 @@ def unfavourite_maintainer(request, maintainer_id):
@require_POST
@ajax_request
def favourite_category(request, category):
packages = Package.objects.for_category(category, last_versions=True)
if not packages:
raise Http404
_, created = CategoryAssociation.objects.get_or_create(
user=request.user, category=category
)
return {"success": created}
obj = Category.objects.get(name=category)
get_profile(request.user).categories.add(obj)
return {"success": True}
@login_required
@require_POST
@ajax_request
def unfavourite_category(request, category):
obj = get_object_or_404(
CategoryAssociation, user=request.user, category=category
)
obj.delete()
obj = Category.objects.get(name=category)
get_profile(request.user).categories.remove(obj)
return {"success": True}
@ -547,22 +505,15 @@ def unfavourite_category(request, category):
@require_POST
@ajax_request
def favourite_overlay(request, overlay):
packages = Package.objects.for_overlay(overlay)
if not packages:
raise Http404
_, created = OverlayAssociation.objects.get_or_create(
user=request.user, overlay=overlay
)
return {"success": created}
obj = Category.objects.get(name=overlay)
get_profile(request.user).overlays.add(obj)
return {"success": True}
@login_required
@require_POST
@ajax_request
def unfavourite_overlay(request, overlay):
obj = get_object_or_404(
OverlayAssociation, user=request.user, overlay=overlay
)
obj.delete()
obj = Category.objects.get(name=overlay)
get_profile(request.user).overlays.remove(obj)
return {"success": True}

2
euscanwww/euscanwww/settings.py
View File

@ -261,3 +261,5 @@ except ImportError, ex:
os.environ['ROOT'] = PORTAGE_ROOT
os.environ['PORTAGE_CONFIGROOT'] = PORTAGE_CONFIGROOT
os.environ['EIX_CACHEFILE'] = EIX_CACHEFILE
AUTH_PROFILE_MODULE = 'djeuscan.UserProfile'

1
euscanwww/runtests.py
View File

@ -29,6 +29,7 @@ settings.configure(
USE_TZ=True,
TASKS_CONCURRENTLY=8,
TASKS_SUBTASK_PACKAGES=32,
AUTH_PROFILE_MODULE="djeuscan.UserProfile"
)

103
pym/euscan/handlers/__init__.py
View File

@ -1,35 +1,86 @@
import pkgutil
# autoimport all modules in this directory and append them to handlers list
handlers = []
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
module = loader.find_module(module_name).load_module(module_name)
handlers.append(module)
# sort handlers by priority (e.g.: generic should be run lastly)
handlers = sorted(
handlers,
key=lambda handler: handler.PRIORITY,
reverse=True
)
import sys
from euscan import CONFIG, output
from euscan.handlers.package import handlers as pkg_handlers
from euscan.handlers.url import handlers as url_handlers
def find_best_handler(pkg, url):
for handler in handlers:
def find_best_pkg_handler(pkg):
"""
Find the best handler for the given package
"""
for handler in pkg_handlers:
if handler.can_handle(pkg):
return handler
return None
def find_best_url_handler(pkg, url):
"""
Find the best handler for the given url
"""
for handler in url_handlers:
if handler.can_handle(pkg, url):
return handler
return None
def scan(pkg, url):
handler = find_best_handler(pkg, url)
if handler:
return handler.scan(pkg, url)
return []
def scan(pkg, urls, on_progress=None):
"""
Scans upstream for the given package.
First tries if a package wide handler is available, then fallbacks
in url handling.
"""
pkg_handler = find_best_pkg_handler(pkg)
if pkg_handler:
if on_progress:
on_progress(increment=35)
if not CONFIG['quiet'] and not CONFIG['format']:
sys.stdout.write("\n")
def brute_force(pkg, url):
handler = find_best_handler(pkg, url)
if handler:
return handler.brute_force(pkg, url)
return []
versions = pkg_handler.scan(pkg)
if on_progress:
on_progress(increment=35)
return versions
if on_progress:
progress_available = 70
num_urls = sum([len(urls[fn]) for fn in urls])
if num_urls > 0:
progress_increment = progress_available / num_urls
else:
progress_increment = 0
versions = []
for filename in urls:
for url in urls[filename]:
if on_progress and progress_available > 0:
on_progress(increment=progress_increment)
progress_available -= progress_increment
if not CONFIG['quiet'] and not CONFIG['format']:
sys.stdout.write("\n")
output.einfo("SRC_URI is '%s'" % url)
if '://' not in url:
output.einfo("Invalid url '%s'" % url)
continue
try:
url_handler = find_best_url_handler(pkg, url)
versions.extend(url_handler.scan(pkg, url))
except Exception as e:
output.ewarn(
"Handler failed: [%s] %s" %
(e.__class__.__name__, e.message)
)
if versions and CONFIG['oneshot']:
break
if on_progress and progress_available > 0:
on_progress(increment=progress_available)
return versions

19
pym/euscan/handlers/package/__init__.py Normal file
View File

@ -0,0 +1,19 @@
"""
Package wide handlers for scanning upstream
"""
import pkgutil
handlers = []
# autoimport all modules in this directory and append them to handlers list
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
module = loader.find_module(module_name).load_module(module_name)
handlers.append(module)
# sort handlers by priority
handlers = sorted(
handlers,
key=lambda handler: handler.PRIORITY,
reverse=True
)

43
pym/euscan/handlers/package/remote_id.py Normal file
View File

@ -0,0 +1,43 @@
from euscan.handlers.url import handlers
from euscan import output
PRIORITY = 100
HANDLER_NAME = "remote_id"
CONFIDENCE = 100.0
url_handlers = {handler.HANDLER_NAME: handler for handler in handlers}
def can_handle(pkg):
# Return True if there's at least one remote-id that can be
# handled by euscan
try:
remoteids = pkg.metadata.upstream()[0].upstream_remoteids()
except IndexError:
pass
else:
if len(remoteids) > 0:
for remote_value, remote_type in remoteids:
if remote_type in url_handlers:
return True
return False
def scan(pkg):
output.einfo("Using remote-id data")
ret = []
remoteids = pkg.metadata.upstream()[0].upstream_remoteids()
for remote_value, remote_type in remoteids:
if remote_type in url_handlers:
remote_data = remote_value.split("/")
scan_remote = getattr(
url_handlers[remote_type], "scan_remote", None
)
if scan_remote:
for url, pv in scan_remote(pkg, remote_data):
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret

14
pym/euscan/handlers/watch.py → pym/euscan/handlers/package/watch.py
View File

@ -3,7 +3,7 @@ import urllib2
import portage
from euscan.handlers import generic
from euscan.handlers.url import generic
from euscan import output, helpers
PRIORITY = 100
@ -15,7 +15,7 @@ CONFIDENCE = 100.0
is_pattern = r"\([^\/]+\)"
def can_handle(pkg, url):
def can_handle(pkg):
try:
return pkg.metadata._xml_tree.find("upstream").find("watch") \
is not None
@ -116,7 +116,7 @@ def handle_directory_patterns(base, file_pattern):
for _, path in scan_data]
def scan(pkg, url):
def scan(pkg):
output.einfo("Using watch data")
cp, ver, rev = portage.pkgsplit(pkg.cpv)
@ -126,18 +126,14 @@ def scan(pkg, url):
if not re.search(is_pattern, base):
steps = [(base, file_pattern)]
res = generic.scan_directory_recursive(
cp, ver, rev, "", steps, url
cp, ver, rev, "", steps, base
)
else:
res = []
for step in handle_directory_patterns(base, file_pattern):
res += generic.scan_directory_recursive(
cp, ver, rev, "", [step], url
cp, ver, rev, "", [step], base
)
results += clean_results(res, versionmangle, urlmangle)
return results
def brute_force(pkg, url):
return []

19
pym/euscan/handlers/url/__init__.py Normal file
View File

@ -0,0 +1,19 @@
"""
Url wide handlers for scanning upstream
"""
import pkgutil
handlers = []
# autoimport all modules in this directory and append them to handlers list
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
module = loader.find_module(module_name).load_module(module_name)
handlers.append(module)
# sort handlers by priority
handlers = sorted(
handlers,
key=lambda handler: handler.PRIORITY,
reverse=True
)

27
pym/euscan/handlers/cpan.py → pym/euscan/handlers/url/cpan.py
View File

@ -81,12 +81,22 @@ def cpan_vercmp(cp, a, b):
def scan(pkg, url):
cp, ver, rev = portage.pkgsplit(pkg.cpv)
pkg = guess_package(cp, url)
remote_pkg = guess_package(cp, url)
orig_url = url
url = 'http://search.cpan.org/api/dist/%s' % pkg
output.einfo("Using CPAN API: %s", remote_pkg)
output.einfo("Using: " + url)
result = scan_remote(pkg, [remote_pkg])
ret = []
for url, pv in result:
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret
def scan_remote(pkg, remote_data):
remote_pkg = remote_data[0]
url = 'http://search.cpan.org/api/dist/%s' % remote_pkg
cp, ver, rev = portage.pkgsplit(pkg.cpv)
try:
fp = helpers.urlopen(url)
@ -125,13 +135,6 @@ def scan(pkg, url):
version['archive']
)
if url == orig_url:
continue
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
ret.append((url, pv))
return ret
def brute_force(pkg, url):
return []

55
pym/euscan/handlers/generic.py → pym/euscan/handlers/url/generic.py
View File

@ -108,43 +108,50 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
def scan(pkg, url):
for bu in SCANDIR_BLACKLIST_URLS:
if re.match(bu, url):
output.einfo("%s is blacklisted by rule %s" % (url, bu))
if CONFIG["scan-dir"]:
for bu in SCANDIR_BLACKLIST_URLS:
if re.match(bu, url):
output.einfo("%s is blacklisted by rule %s" % (url, bu))
return []
resolved_url = helpers.parse_mirror(url)
if not resolved_url:
return []
resolved_url = helpers.parse_mirror(url)
if not resolved_url:
return []
cp, ver, rev = portage.pkgsplit(pkg.cpv)
cp, ver, rev = portage.pkgsplit(pkg.cpv)
# 'Hack' for _beta/_rc versions where _ is used instead of -
if ver not in resolved_url:
newver = helpers.version_change_end_sep(ver)
if newver and newver in resolved_url:
output.einfo(
"Version: using %s instead of %s" % (newver, ver)
)
ver = newver
# 'Hack' for _beta/_rc versions where _ is used instead of -
if ver not in resolved_url:
newver = helpers.version_change_end_sep(ver)
if newver and newver in resolved_url:
template = helpers.template_from_url(resolved_url, ver)
if '${' not in template:
output.einfo(
"Version: using %s instead of %s" % (newver, ver)
"Url doesn't seems to depend on version: %s not found in %s" %
(ver, resolved_url)
)
ver = newver
return []
else:
output.einfo("Scanning: %s" % template)
template = helpers.template_from_url(resolved_url, ver)
if '${' not in template:
output.einfo(
"Url doesn't seems to depend on version: %s not found in %s" %
(ver, resolved_url)
)
return []
else:
output.einfo("Scanning: %s" % template)
steps = helpers.generate_scan_paths(template)
ret = scan_directory_recursive(cp, ver, rev, "", steps, url)
steps = helpers.generate_scan_paths(template)
ret = scan_directory_recursive(cp, ver, rev, "", steps, url)
if not ret:
ret = brute_force(pkg, url)
return ret
def brute_force(pkg, url):
if CONFIG["brute-force"] == 0:
return []
cp, ver, rev = portage.pkgsplit(pkg.cpv)
url = helpers.parse_mirror(url)

7
pym/euscan/handlers/github.py → pym/euscan/handlers/url/github.py
View File

@ -44,6 +44,7 @@ def scan(pkg, url):
(user, project))
dls = json.load(dlreq)
ret = []
for dl in dls:
m = fnre.match(dl['name'])
@ -51,8 +52,6 @@ def scan(pkg, url):
pv = helpers.gentoo_mangle_version(m.group(1))
if helpers.version_filtered(cp, ver, pv):
continue
yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE)
def brute_force(pkg, url):
return []
ret.append((dl['html_url'], pv, HANDLER_NAME, CONFIDENCE))
return ret

15
pym/euscan/handlers/kde.py → pym/euscan/handlers/url/kde.py
View File

@ -1,4 +1,4 @@
from euscan.handlers import generic
from euscan.handlers.url import generic
PRIORITY = 90
@ -29,14 +29,11 @@ def scan(pkg, url):
url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/')
results += generic.scan(pkg.cpv, url)
return clean_results(results)
if not results: # if nothing was found go brute forcing
results = generic.brute_force(pkg.cpv, url)
def brute_force(pkg, url):
results = generic.brute_force(pkg.cpv, url)
if url.startswith('mirror://kde/unstable/'):
url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/')
results += generic.brute_force(pkg.cpv, url)
if url.startswith('mirror://kde/unstable/'):
url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/')
results += generic.brute_force(pkg.cpv, url)
return clean_results(results)

4
pym/euscan/handlers/php.py → pym/euscan/handlers/url/php.py
View File

@ -70,7 +70,3 @@ def scan(pkg, url):
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret
def brute_force(pkg, url):
return []

17
pym/euscan/handlers/pypi.py → pym/euscan/handlers/url/pypi.py
View File

@ -29,6 +29,15 @@ def scan(pkg, url):
package = guess_package(pkg.cpv, url)
ret = []
for urls, pv in scan_remote(pkg, [package]):
ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
return ret
def scan_remote(pkg, remote_data):
package = remote_data[0]
output.einfo("Using PyPi XMLRPC: " + package)
client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
@ -42,17 +51,11 @@ def scan(pkg, url):
cp, ver, rev = portage.pkgsplit(pkg.cpv)
ret = []
for up_pv in versions:
pv = helpers.gentoo_mangle_version(up_pv)
if helpers.version_filtered(cp, ver, pv):
continue
urls = client.release_urls(package, up_pv)
urls = " ".join([infos['url'] for infos in urls])
ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
ret.append((urls, pv))
return ret
def brute_force(pkg, url):
return []

26
pym/euscan/handlers/rubygem.py → pym/euscan/handlers/url/rubygems.py
View File

@ -5,7 +5,7 @@ import urllib2
from euscan import helpers, output
HANDLER_NAME = "rubygem"
HANDLER_NAME = "rubygems"
CONFIDENCE = 100.0
PRIORITY = 90
@ -33,14 +33,23 @@ def scan(pkg, url):
'http://guides.rubygems.org/rubygems-org-api/#gemversion'
gem = guess_gem(pkg.cpv, url)
if not gem:
output.eerror("Can't guess gem name using %s and %s" % \
(pkg.cpv, url))
return []
url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
output.einfo("Using RubyGem API: %s" % gem)
output.einfo("Using: " + url)
ret = []
for url, pv in scan_remote(pkg, [gem]):
ret.append(url, pv, HANDLER_NAME, CONFIDENCE)
return ret
def scan_remote(pkg, remote_data):
gem = remote_data[0]
url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
try:
fp = helpers.urlopen(url)
@ -55,23 +64,14 @@ def scan(pkg, url):
data = fp.read()
versions = json.loads(data)
if not versions:
return []
cp, ver, rev = portage.pkgsplit(pkg.cpv)
ret = []
for version in versions:
up_pv = version['number']
pv = helpers.gentoo_mangle_version(up_pv)
if helpers.version_filtered(cp, ver, pv):
continue
url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
ret.append((url, pv))
return ret
def brute_force(pkg, url):
return []

72
pym/euscan/scan.py
View File

@ -44,60 +44,6 @@ def filter_versions(cp, versions):
]
def scan_upstream_urls(pkg, urls, on_progress):
versions = []
if on_progress:
progress_available = 70
num_urls = sum([len(urls[fn]) for fn in urls])
if num_urls > 0:
progress_increment = progress_available / num_urls
else:
progress_increment = 0
for filename in urls:
for url in urls[filename]:
if on_progress and progress_available > 0:
on_progress(increment=progress_increment)
progress_available -= progress_increment
if not CONFIG['quiet'] and not CONFIG['format']:
pp.uprint()
output.einfo("SRC_URI is '%s'" % url)
if '://' not in url:
output.einfo("Invalid url '%s'" % url)
continue
# Try normal scan
if CONFIG["scan-dir"]:
try:
versions.extend(handlers.scan(pkg, url))
except Exception as e:
output.ewarn("Handler failed: [%s] %s"
% (e.__class__.__name__, e.message))
if versions and CONFIG['oneshot']:
break
# Brute Force
if CONFIG["brute-force"] > 0:
versions.extend(handlers.brute_force(pkg, url))
if versions and CONFIG['oneshot']:
break
cp, ver, rev = portage.pkgsplit(pkg.cpv)
result = filter_versions(cp, versions)
if on_progress and progress_available > 0:
on_progress(increment=progress_available)
return result
# gentoolkit stores PORTDB, so even if we modify it to add an overlay
# it will still use the old dbapi
def reload_gentoolkit():
@ -120,7 +66,6 @@ def scan_upstream(query, on_progress=None):
"""
Scans the upstream searching new versions for the given query
"""
matches = []
if query.endswith(".ebuild"):
@ -188,10 +133,6 @@ def scan_upstream(query, on_progress=None):
output.metadata("description", pkg.environment("DESCRIPTION"))
cpv = pkg.cpv
_, _, ver, _ = portage.catpkgsplit(cpv)
is_current_version_stable = is_version_stable(ver)
metadata = {
"EAPI": portage.settings["EAPI"],
"SRC_URI": pkg.environment("SRC_URI", False),
@ -212,15 +153,20 @@ def scan_upstream(query, on_progress=None):
else:
urls = alist
# output scan time for formatted output
scan_time = (datetime.now() - start_time).total_seconds()
output.metadata("scan_time", scan_time, show=False)
versions = handlers.scan(pkg, urls, on_progress)
result = scan_upstream_urls(pkg, urls, on_progress)
cp, ver, rev = portage.pkgsplit(pkg.cpv)
result = filter_versions(cp, versions)
if on_progress:
on_progress(increment=10)
# output scan time for formatted output
scan_time = (datetime.now() - start_time).total_seconds()
output.metadata("scan_time", scan_time, show=False)
is_current_version_stable = is_version_stable(ver)
if len(result) > 0:
if not (CONFIG['format'] or CONFIG['quiet']):
print("\n", file=sys.stderr)