diff --git a/bin/euscan b/bin/euscan index 47bb7b4..441d479 100755 --- a/bin/euscan +++ b/bin/euscan @@ -129,9 +129,9 @@ def print_usage(_error=None, help=None): " - Ignore non-stable versions only if current\n" + " " * 38 + "version is stable", file=out) print(yellow(" --mirror") + - " - use mirror:// URIs", file=out) + " - use mirror:// URIs", file=out) print(yellow(" --ebuild-uri") + - " - use ebuild variables in URIs", file=out) + " - use ebuild variables in URIs", file=out) print(file=out) if _error in ('packages',) or help: diff --git a/bin/euscan_patch_metadata b/bin/euscan_patch_metadata index 642b639..2f1a834 100755 --- a/bin/euscan_patch_metadata +++ b/bin/euscan_patch_metadata @@ -9,7 +9,7 @@ import tarfile import gzip import logging import shutil -import subprocess +from difflib import unified_diff from portage.exception import AmbiguousPackageName from gentoolkit.query import Query @@ -117,7 +117,7 @@ def get_deb_url(name): content = opened.read() for link in BeautifulSoup(content, parseOnlyThese=SoupStrainer("a")): - if re.match("[^\s]+\.debian\.tar\.gz", link.text): + if re.match("[^\s]+\.debian\.tar\.(?:gz|bz2)", link.text): deb_url = link["href"] deb_type = "source" break @@ -133,9 +133,11 @@ def get_deb_url(name): return deb_url, deb_type -def patch_metadata(metadata_path, watch_data, diff=False): +def patch_metadata(package, watch_data, diff=False): logger.info(" Patching metadata file") + metadata_path = package.metadata.metadata_path + with open(metadata_path) as fp: original = fp.read() rindent, indent = guess_indent_values(original) @@ -203,14 +205,17 @@ def patch_metadata(metadata_path, watch_data, diff=False): if not diff: return data else: - _, data_path = mkstemp() - with open(data_path, "w") as f: - f.write(data) - sub = subprocess.Popen(["diff", metadata_path, data_path], - stdout=subprocess.PIPE) - content = sub.stdout.read() - os.unlink(data_path) - return content + # Generate clean a/category/package/metadata.xml path + n = metadata_path.find(package.category) + if n != -1: + metadata_path = metadata_path[n:] + res = unified_diff( + original.splitlines(True), + data.splitlines(True), + fromfile=os.path.join('a/', metadata_path), + tofile=os.path.join('b/', metadata_path), + ) + return "".join([x for x in res]) def process_package(query, diff=False): @@ -236,12 +241,11 @@ def process_package(query, diff=False): if '9999' in package.version and len(matches) > 0: package = matches.pop() - metadata_path = package.metadata.metadata_path watch_data = get_watch_data(package) if watch_data is None: logger.error(" No watch file found") else: - return patch_metadata(metadata_path, watch_data, diff=diff) + return patch_metadata(package, watch_data, diff=diff) def main(): @@ -261,7 +265,7 @@ def main(): logger.info("Processing %s..." % package) result = process_package(package, opts.diff) if result: - print result + sys.stdout.write(result) if __name__ == "__main__": main() diff --git a/euscanwww/djeuscan/admin.py b/euscanwww/djeuscan/admin.py index 0322e18..a24870e 100644 --- a/euscanwww/djeuscan/admin.py +++ b/euscanwww/djeuscan/admin.py @@ -1,7 +1,6 @@ from djeuscan.models import Package, Version, VersionLog, EuscanResult, \ Log, WorldLog, CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer, \ - RefreshPackageQuery, HerdAssociation, CategoryAssociation, \ - MaintainerAssociation, PackageAssociation, ProblemReport + RefreshPackageQuery, Category, Overlay, ProblemReport from django.contrib import admin @@ -54,9 +53,7 @@ admin.site.register(HerdLog) admin.site.register(MaintainerLog) admin.site.register(RefreshPackageQuery) -admin.site.register(HerdAssociation) -admin.site.register(CategoryAssociation) -admin.site.register(MaintainerAssociation) -admin.site.register(PackageAssociation) +admin.site.register(Category) +admin.site.register(Overlay) admin.site.register(ProblemReport, ProblemReportAdmin) diff --git a/euscanwww/djeuscan/helpers.py b/euscanwww/djeuscan/helpers.py index f7a6b82..c05daaf 100644 --- a/euscanwww/djeuscan/helpers.py +++ b/euscanwww/djeuscan/helpers.py @@ -76,33 +76,33 @@ class catch_and_return(object): return wrapper -def get_account_categories(user): - from djeuscan.models import Package, CategoryAssociation +def get_profile(user): + from djeuscan.models import UserProfile + try: + return user.get_profile() + except UserProfile.DoesNotExist: + UserProfile.objects.create(user=user) + return user.get_profile() + +def get_account_categories(user): + from djeuscan.models import Package # TODO: This is quite ugly - category_names = [obj.category for obj in - CategoryAssociation.objects.filter(user=user)] + category_names = [obj.name for obj in get_profile(user).categories.all()] return [c for c in Package.objects.categories() if c["category"] in category_names] def get_account_herds(user): - from djeuscan.models import Package, HerdAssociation + from djeuscan.models import Package - ids = [obj.herd.pk for obj in - HerdAssociation.objects.filter(user=user)] + ids = [herd.pk for herd in get_profile(user).herds.all()] return Package.objects.herds(ids=ids) def get_account_maintainers(user): - from djeuscan.models import Package, MaintainerAssociation + from djeuscan.models import Package - ids = [obj.maintainer.pk for obj in - MaintainerAssociation.objects.filter(user=user)] + ids = [obj.pk for obj in get_profile(user).maintainers.all()] return Package.objects.maintainers(ids=ids) - -def get_account_packages(user): - from djeuscan.models import PackageAssociation - return [obj.package for obj in - PackageAssociation.objects.filter(user=user)] diff --git a/euscanwww/djeuscan/migrations/0016_auto__del_maintainerassociation__del_unique_maintainerassociation_user.py b/euscanwww/djeuscan/migrations/0016_auto__del_maintainerassociation__del_unique_maintainerassociation_user.py new file mode 100644 index 0000000..b4fe300 --- /dev/null +++ b/euscanwww/djeuscan/migrations/0016_auto__del_maintainerassociation__del_unique_maintainerassociation_user.py @@ -0,0 +1,355 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Removing unique constraint on 'CategoryAssociation', fields ['user', 'category'] + db.delete_unique('djeuscan_categoryassociation', ['user_id', 'category']) + + # Removing unique constraint on 'OverlayAssociation', fields ['user', 'overlay'] + db.delete_unique('djeuscan_overlayassociation', ['user_id', 'overlay']) + + # Removing unique constraint on 'HerdAssociation', fields ['user', 'herd'] + db.delete_unique('djeuscan_herdassociation', ['user_id', 'herd_id']) + + # Removing unique constraint on 'PackageAssociation', fields ['user', 'package'] + db.delete_unique('djeuscan_packageassociation', ['user_id', 'package_id']) + + # Removing unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer'] + db.delete_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id']) + + # Deleting model 'MaintainerAssociation' + db.delete_table('djeuscan_maintainerassociation') + + # Deleting model 'PackageAssociation' + db.delete_table('djeuscan_packageassociation') + + # Deleting model 'HerdAssociation' + db.delete_table('djeuscan_herdassociation') + + # Deleting model 'OverlayAssociation' + db.delete_table('djeuscan_overlayassociation') + + # Deleting model 'CategoryAssociation' + db.delete_table('djeuscan_categoryassociation') + + # Adding model 'UserProfile' + db.create_table('djeuscan_userprofile', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('user', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['auth.User'], unique=True)), + )) + db.send_create_signal('djeuscan', ['UserProfile']) + + # Adding M2M table for field herds on 'UserProfile' + db.create_table('djeuscan_userprofile_herds', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)), + ('herd', models.ForeignKey(orm['djeuscan.herd'], null=False)) + )) + db.create_unique('djeuscan_userprofile_herds', ['userprofile_id', 'herd_id']) + + # Adding M2M table for field maintainers on 'UserProfile' + db.create_table('djeuscan_userprofile_maintainers', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)), + ('maintainer', models.ForeignKey(orm['djeuscan.maintainer'], null=False)) + )) + db.create_unique('djeuscan_userprofile_maintainers', ['userprofile_id', 'maintainer_id']) + + # Adding M2M table for field packages on 'UserProfile' + db.create_table('djeuscan_userprofile_packages', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)), + ('package', models.ForeignKey(orm['djeuscan.package'], null=False)) + )) + db.create_unique('djeuscan_userprofile_packages', ['userprofile_id', 'package_id']) + + # Adding M2M table for field categories on 'UserProfile' + db.create_table('djeuscan_userprofile_categories', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)), + ('category', models.ForeignKey(orm['djeuscan.category'], null=False)) + )) + db.create_unique('djeuscan_userprofile_categories', ['userprofile_id', 'category_id']) + + # Adding M2M table for field overlays on 'UserProfile' + db.create_table('djeuscan_userprofile_overlays', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)), + ('overlay', models.ForeignKey(orm['djeuscan.overlay'], null=False)) + )) + db.create_unique('djeuscan_userprofile_overlays', ['userprofile_id', 'overlay_id']) + + # Adding model 'Category' + db.create_table('djeuscan_category', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128)), + )) + db.send_create_signal('djeuscan', ['Category']) + + # Adding model 'Overlay' + db.create_table('djeuscan_overlay', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128)), + )) + db.send_create_signal('djeuscan', ['Overlay']) + + def backwards(self, orm): + # Adding model 'MaintainerAssociation' + db.create_table('djeuscan_maintainerassociation', ( + ('maintainer', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Maintainer'])), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + )) + db.send_create_signal('djeuscan', ['MaintainerAssociation']) + + # Adding unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer'] + db.create_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id']) + + # Adding model 'PackageAssociation' + db.create_table('djeuscan_packageassociation', ( + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('package', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Package'])), + )) + db.send_create_signal('djeuscan', ['PackageAssociation']) + + # Adding unique constraint on 'PackageAssociation', fields ['user', 'package'] + db.create_unique('djeuscan_packageassociation', ['user_id', 'package_id']) + + # Adding model 'HerdAssociation' + db.create_table('djeuscan_herdassociation', ( + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('herd', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Herd'])), + )) + db.send_create_signal('djeuscan', ['HerdAssociation']) + + # Adding unique constraint on 'HerdAssociation', fields ['user', 'herd'] + db.create_unique('djeuscan_herdassociation', ['user_id', 'herd_id']) + + # Adding model 'OverlayAssociation' + db.create_table('djeuscan_overlayassociation', ( + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('overlay', self.gf('django.db.models.fields.CharField')(max_length=128)), + )) + db.send_create_signal('djeuscan', ['OverlayAssociation']) + + # Adding unique constraint on 'OverlayAssociation', fields ['user', 'overlay'] + db.create_unique('djeuscan_overlayassociation', ['user_id', 'overlay']) + + # Adding model 'CategoryAssociation' + db.create_table('djeuscan_categoryassociation', ( + ('category', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + )) + db.send_create_signal('djeuscan', ['CategoryAssociation']) + + # Adding unique constraint on 'CategoryAssociation', fields ['user', 'category'] + db.create_unique('djeuscan_categoryassociation', ['user_id', 'category']) + + # Deleting model 'UserProfile' + db.delete_table('djeuscan_userprofile') + + # Removing M2M table for field herds on 'UserProfile' + db.delete_table('djeuscan_userprofile_herds') + + # Removing M2M table for field maintainers on 'UserProfile' + db.delete_table('djeuscan_userprofile_maintainers') + + # Removing M2M table for field packages on 'UserProfile' + db.delete_table('djeuscan_userprofile_packages') + + # Removing M2M table for field categories on 'UserProfile' + db.delete_table('djeuscan_userprofile_categories') + + # Removing M2M table for field overlays on 'UserProfile' + db.delete_table('djeuscan_userprofile_overlays') + + # Deleting model 'Category' + db.delete_table('djeuscan_category') + + # Deleting model 'Overlay' + db.delete_table('djeuscan_overlay') + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'djeuscan.category': { + 'Meta': {'object_name': 'Category'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.categorylog': { + 'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.euscanresult': { + 'Meta': {'object_name': 'EuscanResult'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}) + }, + 'djeuscan.herd': { + 'Meta': {'object_name': 'Herd'}, + 'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'}) + }, + 'djeuscan.herdlog': { + 'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.log': { + 'Meta': {'object_name': 'Log'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'djeuscan.maintainer': { + 'Meta': {'object_name': 'Maintainer'}, + 'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.maintainerlog': { + 'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}) + }, + 'djeuscan.overlay': { + 'Meta': {'object_name': 'Overlay'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.package': { + 'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}), + 'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}), + 'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.problemreport': { + 'Meta': {'object_name': 'ProblemReport'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Version']", 'null': 'True', 'blank': 'True'}) + }, + 'djeuscan.refreshpackagequery': { + 'Meta': {'object_name': 'RefreshPackageQuery'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'symmetrical': 'False'}) + }, + 'djeuscan.userprofile': { + 'Meta': {'object_name': 'UserProfile'}, + 'categories': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Category']", 'symmetrical': 'False'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'}), + 'overlays': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Overlay']", 'symmetrical': 'False'}), + 'packages': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Package']", 'symmetrical': 'False'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'}) + }, + 'djeuscan.version': { + 'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'}, + 'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'ebuild_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'metadata_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.versionlog': { + 'Meta': {'object_name': 'VersionLog'}, + 'action': ('django.db.models.fields.IntegerField', [], {}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.worldlog': { + 'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + } + } + + complete_apps = ['djeuscan'] \ No newline at end of file diff --git a/euscanwww/djeuscan/migrations/0017_auto__add_unique_overlay_name__add_unique_category_name.py b/euscanwww/djeuscan/migrations/0017_auto__add_unique_overlay_name__add_unique_category_name.py new file mode 100644 index 0000000..7d95034 --- /dev/null +++ b/euscanwww/djeuscan/migrations/0017_auto__add_unique_overlay_name__add_unique_category_name.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding unique constraint on 'Overlay', fields ['name'] + db.create_unique('djeuscan_overlay', ['name']) + + # Adding unique constraint on 'Category', fields ['name'] + db.create_unique('djeuscan_category', ['name']) + + def backwards(self, orm): + # Removing unique constraint on 'Category', fields ['name'] + db.delete_unique('djeuscan_category', ['name']) + + # Removing unique constraint on 'Overlay', fields ['name'] + db.delete_unique('djeuscan_overlay', ['name']) + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'djeuscan.category': { + 'Meta': {'object_name': 'Category'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}) + }, + 'djeuscan.categorylog': { + 'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.euscanresult': { + 'Meta': {'object_name': 'EuscanResult'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}) + }, + 'djeuscan.herd': { + 'Meta': {'object_name': 'Herd'}, + 'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'}) + }, + 'djeuscan.herdlog': { + 'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.log': { + 'Meta': {'object_name': 'Log'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'djeuscan.maintainer': { + 'Meta': {'object_name': 'Maintainer'}, + 'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.maintainerlog': { + 'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}) + }, + 'djeuscan.overlay': { + 'Meta': {'object_name': 'Overlay'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}) + }, + 'djeuscan.package': { + 'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}), + 'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}), + 'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.problemreport': { + 'Meta': {'object_name': 'ProblemReport'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Version']", 'null': 'True', 'blank': 'True'}) + }, + 'djeuscan.refreshpackagequery': { + 'Meta': {'object_name': 'RefreshPackageQuery'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'symmetrical': 'False'}) + }, + 'djeuscan.userprofile': { + 'Meta': {'object_name': 'UserProfile'}, + 'categories': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Category']", 'symmetrical': 'False'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'}), + 'overlays': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Overlay']", 'symmetrical': 'False'}), + 'packages': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Package']", 'symmetrical': 'False'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'}) + }, + 'djeuscan.version': { + 'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'}, + 'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'ebuild_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'metadata_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.versionlog': { + 'Meta': {'object_name': 'VersionLog'}, + 'action': ('django.db.models.fields.IntegerField', [], {}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.worldlog': { + 'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + } + } + + complete_apps = ['djeuscan'] \ No newline at end of file diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py index 6626a41..86f79bc 100644 --- a/euscanwww/djeuscan/models.py +++ b/euscanwww/djeuscan/models.py @@ -231,6 +231,31 @@ class EuscanResult(models.Model): ) +class Category(models.Model): + name = models.CharField(max_length=128, validators=[validate_category], + unique=True) + + def __unicode__(self): + return self.name + + +class Overlay(models.Model): + name = models.CharField(max_length=128, validators=[validate_name], + unique=True) + + def __unicode__(self): + return self.name + + +class UserProfile(models.Model): + user = models.OneToOneField(User) + herds = models.ManyToManyField(Herd) + maintainers = models.ManyToManyField(Maintainer) + packages = models.ManyToManyField(Package) + categories = models.ManyToManyField(Category) + overlays = models.ManyToManyField(Overlay) + + class Log(models.Model): """ Model used for keeping data for charts @@ -303,61 +328,6 @@ class RefreshPackageQuery(models.Model): return u'[%d] %s' % (self.priority, self.package) -class HerdAssociation(models.Model): - user = models.ForeignKey(User) - herd = models.ForeignKey(Herd) - - class Meta: - unique_together = ['user', 'herd'] - - def __unicode__(self): - return u'[%s] %s' % (self.user, self.herd) - - -class MaintainerAssociation(models.Model): - user = models.ForeignKey(User) - maintainer = models.ForeignKey(Maintainer) - - class Meta: - unique_together = ['user', 'maintainer'] - - def __unicode__(self): - return u'[%s] %s' % (self.user, self.maintainer) - - -class PackageAssociation(models.Model): - user = models.ForeignKey(User) - package = models.ForeignKey(Package) - - class Meta: - unique_together = ['user', 'package'] - - def __unicode__(self): - return u'[%s] %s' % (self.user, self.package) - - -class CategoryAssociation(models.Model): - user = models.ForeignKey(User) - category = models.CharField(max_length=128, validators=[validate_category]) - - class Meta: - unique_together = ['user', 'category'] - - def __unicode__(self): - return u'[%s] %s' % (self.user, self.category) - - -class OverlayAssociation(models.Model): - user = models.ForeignKey(User) - overlay = models.CharField(max_length=128, validators=[validate_name]) - - class Meta: - unique_together = ['user', 'overlay'] - - def __unicode__(self): - return u'[%s] %s' % (self.user, self.category) - - class ProblemReport(models.Model): package = models.ForeignKey(Package) version = models.ForeignKey(Version, null=True, blank=True) diff --git a/euscanwww/djeuscan/processing/misc/update_counters.py b/euscanwww/djeuscan/processing/misc/update_counters.py index 09ba1db..02d9b4b 100644 --- a/euscanwww/djeuscan/processing/misc/update_counters.py +++ b/euscanwww/djeuscan/processing/misc/update_counters.py @@ -1,8 +1,8 @@ from django.db.transaction import commit_on_success from django.utils import timezone -from djeuscan.models import Package, Herd, Maintainer, Version -from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog +from djeuscan.models import Package, Herd, Maintainer, Version, HerdLog, \ + MaintainerLog, CategoryLog, WorldLog, Category, Overlay from djeuscan import charts from djeuscan.processing import FakeLogger @@ -48,6 +48,19 @@ def update_counters(fast=False, nolog=False, logger=None): wlog = None + # Populate Category and Overlay + for cat in Package.objects.values('category').distinct(): + obj, created = Category.objects.get_or_create(name=cat["category"]) + if created: + logger.info("+ [c] %s", cat["category"]) + + for overlay in Version.objects.values('overlay').distinct(): + if not overlay["overlay"]: + continue + obj, created = Overlay.objects.get_or_create(name=overlay["overlay"]) + if created: + logger.info("+ [o] %s", overlay["overlay"]) + if not nolog: wlog = WorldLog() wlog.datetime = now @@ -157,17 +170,17 @@ def update_counters(fast=False, nolog=False, logger=None): return for clog in categories.values(): - logger.info('+ [cl] %s' % clog) + logger.info('+ [cl] %s', clog) charts.rrd_update('category-%s' % clog.category, now, clog) clog.save() for hlog in herds.values(): - logger.info('+ [hl] %s' % hlog) + logger.info('+ [hl] %s', hlog) charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog) hlog.save() for mlog in maintainers.values(): - logger.info('+ [ml] %s' % mlog) + logger.info('+ [ml] %s', mlog) charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog) mlog.save() diff --git a/euscanwww/djeuscan/tests/euscan_factory.py b/euscanwww/djeuscan/tests/euscan_factory.py index 114e3a4..db53a32 100644 --- a/euscanwww/djeuscan/tests/euscan_factory.py +++ b/euscanwww/djeuscan/tests/euscan_factory.py @@ -4,10 +4,12 @@ from datetime import datetime from collections import defaultdict from django.contrib.auth.models import User +from django.db import IntegrityError import factory -from djeuscan.models import Herd, Maintainer, Package, Version, EuscanResult +from djeuscan.models import Herd, Maintainer, Package, Version, EuscanResult, \ + Category, Overlay class UserFactory(factory.Factory): @@ -36,6 +38,18 @@ class MaintainerFactory(factory.Factory): email = factory.LazyAttribute(lambda a: "%s@example.com" % a.name) +class CategoryFactory(factory.Factory): + FACTORY_FOR = Category + + name = factory.LazyAttribute(lambda a: random_string()) + + +class OverlayFactory(factory.Factory): + FACTORY_FOR = Overlay + + name = factory.LazyAttribute(lambda a: random_string()) + + class PackageFactory(factory.Factory): FACTORY_FOR = Package @@ -46,6 +60,16 @@ class PackageFactory(factory.Factory): description = "This is a test package" homepage = "http://testpackage.com" + @classmethod + def _prepare(cls, create, **kwargs): + package = super(PackageFactory, cls)._prepare(create, **kwargs) + category = kwargs.pop('category', None) + if create: + CategoryFactory.create(name=category) + else: + CategoryFactory.build(name=category) + return package + class VersionFactory(factory.Factory): FACTORY_FOR = Version @@ -59,6 +83,19 @@ class VersionFactory(factory.Factory): urls = "http://packageurl.com" alive = True + @classmethod + def _prepare(cls, create, **kwargs): + version = super(VersionFactory, cls)._prepare(create, **kwargs) + overlay = kwargs.pop('overlay', None) + try: + if create: + OverlayFactory.create(name=overlay) + else: + OverlayFactory.build(name=overlay) + except IntegrityError: + pass + return version + class EuscanResultFactory(factory.Factory): FACTORY_FOR = EuscanResult diff --git a/euscanwww/djeuscan/tests/test_views.py b/euscanwww/djeuscan/tests/test_views.py index 4d9ce4b..e9a7877 100644 --- a/euscanwww/djeuscan/tests/test_views.py +++ b/euscanwww/djeuscan/tests/test_views.py @@ -5,13 +5,12 @@ try: except ImportError: from bs4 import BeautifulSoup +from djeuscan.helpers import get_profile + from djeuscan.tests import SystemTestCase from djeuscan.tests.euscan_factory import PackageFactory, setup_maintainers, \ setup_herds, setup_categories, setup_overlays -from djeuscan.models import PackageAssociation, CategoryAssociation, \ - HerdAssociation, MaintainerAssociation - class PagesTest(SystemTestCase): """ @@ -50,11 +49,10 @@ class PackageTests(SystemTestCase): self.assertEqual(response.status_code, 200) def test_favourite(self): - self.assertEqual(PackageAssociation.objects.count(), 0) - response = self.get("package", category=self.package.category, package=self.package.name) self.assertEqual(response.status_code, 200) + self.assertNotIn("Watch", response.content) with self.login(): @@ -62,11 +60,14 @@ class PackageTests(SystemTestCase): package=self.package.name) self.assertEqual(response.status_code, 200) + user = response.context["user"] + self.assertEquals(get_profile(user).categories.count(), 0) + self.assertIn("Watch", response.content) self.post("favourite_package", category=self.package.category, package=self.package.name) - self.assertEqual(PackageAssociation.objects.count(), 1) + self.assertEquals(get_profile(user).packages.count(), 1) response = self.get("accounts_packages") self.assertEqual(response.status_code, 200) @@ -75,7 +76,7 @@ class PackageTests(SystemTestCase): self.post("unfavourite_package", category=self.package.category, package=self.package.name) - self.assertEqual(PackageAssociation.objects.count(), 0) + self.assertEquals(get_profile(user).packages.count(), 0) class SectionTests(SystemTestCase): @@ -118,9 +119,9 @@ class CategoriesTests(SectionTests): def test_favourite(self): category = self.categories[0] - self.assertEqual(CategoryAssociation.objects.count(), 0) response = self.get("category", category=category) + self.assertEqual(response.status_code, 200) self.assertNotIn("Watch", response.content) @@ -128,10 +129,13 @@ class CategoriesTests(SectionTests): response = self.get("category", category=category) self.assertEqual(response.status_code, 200) + user = response.context["user"] + self.assertEquals(get_profile(user).categories.count(), 0) + self.assertIn("Watch", response.content) self.post("favourite_category", category=category) - self.assertEqual(CategoryAssociation.objects.count(), 1) + self.assertEquals(get_profile(user).categories.count(), 1) response = self.get("accounts_categories") self.assertEqual(response.status_code, 200) @@ -139,7 +143,8 @@ class CategoriesTests(SectionTests): self._check_table(response, [category]) self.post("unfavourite_category", category=category) - self.assertEqual(CategoryAssociation.objects.count(), 0) + + self.assertEquals(get_profile(user).categories.count(), 0) class HerdsTests(SectionTests): @@ -167,7 +172,6 @@ class HerdsTests(SectionTests): def test_favourite(self): herd = self.herds[0] - self.assertEqual(HerdAssociation.objects.count(), 0) response = self.get("herd", herd=herd.herd) self.assertEqual(response.status_code, 200) @@ -177,10 +181,13 @@ class HerdsTests(SectionTests): response = self.get("herd", herd=herd.herd) self.assertEqual(response.status_code, 200) + user = response.context["user"] + self.assertEquals(get_profile(user).herds.count(), 0) + self.assertIn("Watch", response.content) self.post("favourite_herd", herd=herd.herd) - self.assertEqual(HerdAssociation.objects.count(), 1) + self.assertEquals(get_profile(user).herds.count(), 1) response = self.get("accounts_herds") self.assertEqual(response.status_code, 200) @@ -188,7 +195,7 @@ class HerdsTests(SectionTests): self._check_table(response, [herd], attr="herd") self.post("unfavourite_herd", herd=herd.herd) - self.assertEqual(HerdAssociation.objects.count(), 0) + self.assertEquals(get_profile(user).herds.count(), 0) class MaintainersTests(SectionTests): @@ -216,7 +223,6 @@ class MaintainersTests(SectionTests): def test_favourite(self): maintainer = self.maintainers[0] - self.assertEqual(MaintainerAssociation.objects.count(), 0) response = self.get("maintainer", maintainer_id=maintainer.pk) self.assertEqual(response.status_code, 200) @@ -226,10 +232,13 @@ class MaintainersTests(SectionTests): response = self.get("maintainer", maintainer_id=maintainer.pk) self.assertEqual(response.status_code, 200) + user = response.context["user"] + self.assertEquals(get_profile(user).maintainers.count(), 0) + self.assertIn("Watch", response.content) self.post("favourite_maintainer", maintainer_id=maintainer.pk) - self.assertEqual(MaintainerAssociation.objects.count(), 1) + self.assertEquals(get_profile(user).maintainers.count(), 1) response = self.get("accounts_maintainers") self.assertEqual(response.status_code, 200) @@ -237,7 +246,8 @@ class MaintainersTests(SectionTests): self._check_table(response, [maintainer], attr="name") self.post("unfavourite_maintainer", maintainer_id=maintainer.pk) - self.assertEqual(MaintainerAssociation.objects.count(), 0) + + self.assertEquals(get_profile(user).maintainers.count(), 0) class OverlayTests(SectionTests): diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py index c9abfca..89460f4 100644 --- a/euscanwww/djeuscan/views.py +++ b/euscanwww/djeuscan/views.py @@ -8,12 +8,10 @@ from django.shortcuts import get_object_or_404 from django.contrib.auth.decorators import login_required from django.views.decorators.http import require_POST -from djeuscan.helpers import version_key, packages_from_names, \ - get_account_categories, get_account_herds, get_account_maintainers, \ - get_account_packages +from djeuscan.helpers import version_key, packages_from_names, get_profile, \ + get_account_categories, get_account_herds, get_account_maintainers from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \ - VersionLog, RefreshPackageQuery, HerdAssociation, MaintainerAssociation, \ - CategoryAssociation, PackageAssociation, OverlayAssociation, ProblemReport + VersionLog, RefreshPackageQuery, ProblemReport, Category, Overlay from djeuscan.forms import WorldForm, PackagesForm, ProblemReportForm from djeuscan.tasks import admin_tasks from djeuscan import charts @@ -68,12 +66,8 @@ def category(request, category): favourited = False if request.user.is_authenticated(): - try: - CategoryAssociation.objects.get(user=request.user, - category=category) - except CategoryAssociation.DoesNotExist: - pass - else: + if Category.objects.get(name=category) in \ + get_profile(request.user).categories.all(): favourited = True return {'category': category, 'packages': packages, 'last_scan': last_scan, @@ -104,11 +98,7 @@ def herd(request, herd): favourited = False if request.user.is_authenticated(): - try: - HerdAssociation.objects.get(user=request.user, herd=herd) - except HerdAssociation.DoesNotExist: - pass - else: + if herd in get_profile(request.user).herds.all(): favourited = True return {'herd': herd, 'packages': packages, "last_scan": last_scan, @@ -140,12 +130,7 @@ def maintainer(request, maintainer_id): favourited = False if request.user.is_authenticated(): - try: - MaintainerAssociation.objects.get(user=request.user, - maintainer=maintainer) - except MaintainerAssociation.DoesNotExist: - pass - else: + if maintainer in get_profile(request.user).maintainers.all(): favourited = True return {'maintainer': maintainer, 'packages': packages, @@ -177,16 +162,14 @@ def overlay(request, overlay): favourited = False if request.user.is_authenticated(): - try: - OverlayAssociation.objects.get(user=request.user, overlay=overlay) - except OverlayAssociation.DoesNotExist: - pass - else: + if Overlay.objects.get(name=overlay) in \ + get_profile(request.user).overlays.all(): favourited = True return {'overlay': overlay, 'packages': packages, 'last_scan': last_scan, 'favourited': favourited} + @render_to('euscan/package.html') def package(request, category, package): package = get_object_or_404(Package, category=category, name=package) @@ -208,11 +191,7 @@ def package(request, category, package): favourited = False if request.user.is_authenticated(): - try: - PackageAssociation.objects.get(user=request.user, package=package) - except PackageAssociation.DoesNotExist: - pass - else: + if package in get_profile(request.user).packages.all(): favourited = True try: @@ -225,14 +204,15 @@ def package(request, category, package): 'package': package, 'packaged': packaged, 'upstream': upstream, - 'log': log.messages(), 'vlog': vlog, - 'msg' : log.messages() if log else "", + 'log': log, + 'msg': log.messages() if log else "", 'last_scan': last_scan, 'favourited': favourited, 'refreshed': refreshed, } + @login_required @render_to('euscan/problem.html') def problem(request, category, package): @@ -265,8 +245,9 @@ def problem(request, category, package): 'package': package, 'packaged': packaged, 'upstream': upstream, - 'msg' : log.messages() if log else "", - } + 'msg': log.messages() if log else "", + } + @render_to('euscan/world.html') def world(request): @@ -276,7 +257,7 @@ def world(request): return { 'world_form': world_form, 'packages_form': packages_form - } + } @render_to('euscan/world_scan.html') @@ -394,9 +375,10 @@ def refresh_package(request, category, package): @login_required @render_to('euscan/accounts/index.html') def accounts_index(request): + user = request.user upstream_k = lambda c: c["n_versions"] - c["n_packaged"] - categories = sorted(get_account_categories(request.user), + categories = sorted(get_account_categories(user), key=upstream_k, reverse=True) c_upstream = sum([c["n_versions"] - c["n_packaged"] for c in categories]) herds = sorted(get_account_herds(request.user), @@ -405,7 +387,7 @@ def accounts_index(request): maintainers = sorted(get_account_maintainers(request.user), key=upstream_k, reverse=True) m_upstream = sum([c["n_versions"] - c["n_packaged"] for c in maintainers]) - packages = sorted(get_account_packages(request.user), + packages = sorted(get_profile(user).packages.all(), key=lambda p: p.n_versions - p.n_packaged, reverse=True) p_upstream = sum([c.n_versions - c.n_packaged for c in packages]) return { @@ -437,14 +419,13 @@ def accounts_maintainers(request): @login_required @render_to('euscan/accounts/packages.html') def accounts_packages(request): - return {"packages": get_account_packages(request.user)} + return {"packages": get_profile(request.user).packages.all()} @login_required @render_to('euscan/accounts/overlays.html') def accounts_overlays(request): - overlays = [obj.overlay for obj in - OverlayAssociation.objects.filter(user=request.user)] + overlays = [obj.name for obj in get_profile(request.user).overlays.all()] return {"overlays": overlays} @@ -453,10 +434,8 @@ def accounts_overlays(request): @ajax_request def favourite_package(request, category, package): obj = get_object_or_404(Package, category=category, name=package) - _, created = PackageAssociation.objects.get_or_create( - user=request.user, package=obj - ) - return {"success": created} + get_profile(request.user).packages.add(obj) + return {"success": True} @login_required @@ -464,10 +443,7 @@ def favourite_package(request, category, package): @ajax_request def unfavourite_package(request, category, package): package = get_object_or_404(Package, category=category, name=package) - obj = get_object_or_404( - PackageAssociation, package=package, user=request.user - ) - obj.delete() + get_profile(request.user).packages.remove(package) return {"success": True} @@ -476,10 +452,8 @@ def unfavourite_package(request, category, package): @ajax_request def favourite_herd(request, herd): obj = get_object_or_404(Herd, herd=herd) - _, created = HerdAssociation.objects.get_or_create( - user=request.user, herd=obj - ) - return {"success": created} + get_profile(request.user).herds.add(obj) + return {"success": True} @login_required @@ -487,10 +461,7 @@ def favourite_herd(request, herd): @ajax_request def unfavourite_herd(request, herd): herd = get_object_or_404(Herd, herd=herd) - obj = get_object_or_404( - HerdAssociation, herd=herd, user=request.user - ) - obj.delete() + get_profile(request.user).herds.remove(herd) return {"success": True} @@ -499,10 +470,8 @@ def unfavourite_herd(request, herd): @ajax_request def favourite_maintainer(request, maintainer_id): obj = get_object_or_404(Maintainer, pk=maintainer_id) - _, created = MaintainerAssociation.objects.get_or_create( - user=request.user, maintainer=obj - ) - return {"success": created} + get_profile(request.user).maintainers.add(obj) + return {"success": True} @login_required @@ -510,10 +479,7 @@ def favourite_maintainer(request, maintainer_id): @ajax_request def unfavourite_maintainer(request, maintainer_id): maintainer = get_object_or_404(Maintainer, pk=maintainer_id) - obj = get_object_or_404( - MaintainerAssociation, maintainer=maintainer, user=request.user - ) - obj.delete() + get_profile(request.user).maintainers.remove(maintainer) return {"success": True} @@ -521,25 +487,17 @@ def unfavourite_maintainer(request, maintainer_id): @require_POST @ajax_request def favourite_category(request, category): - packages = Package.objects.for_category(category, last_versions=True) - - if not packages: - raise Http404 - - _, created = CategoryAssociation.objects.get_or_create( - user=request.user, category=category - ) - return {"success": created} + obj = Category.objects.get(name=category) + get_profile(request.user).categories.add(obj) + return {"success": True} @login_required @require_POST @ajax_request def unfavourite_category(request, category): - obj = get_object_or_404( - CategoryAssociation, user=request.user, category=category - ) - obj.delete() + obj = Category.objects.get(name=category) + get_profile(request.user).categories.remove(obj) return {"success": True} @@ -547,22 +505,15 @@ def unfavourite_category(request, category): @require_POST @ajax_request def favourite_overlay(request, overlay): - packages = Package.objects.for_overlay(overlay) - if not packages: - raise Http404 - - _, created = OverlayAssociation.objects.get_or_create( - user=request.user, overlay=overlay - ) - return {"success": created} + obj = Category.objects.get(name=overlay) + get_profile(request.user).overlays.add(obj) + return {"success": True} @login_required @require_POST @ajax_request def unfavourite_overlay(request, overlay): - obj = get_object_or_404( - OverlayAssociation, user=request.user, overlay=overlay - ) - obj.delete() + obj = Category.objects.get(name=overlay) + get_profile(request.user).overlays.remove(obj) return {"success": True} diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py index fc6b668..d70c99c 100644 --- a/euscanwww/euscanwww/settings.py +++ b/euscanwww/euscanwww/settings.py @@ -261,3 +261,5 @@ except ImportError, ex: os.environ['ROOT'] = PORTAGE_ROOT os.environ['PORTAGE_CONFIGROOT'] = PORTAGE_CONFIGROOT os.environ['EIX_CACHEFILE'] = EIX_CACHEFILE + +AUTH_PROFILE_MODULE = 'djeuscan.UserProfile' diff --git a/euscanwww/runtests.py b/euscanwww/runtests.py index 3721076..a436a27 100644 --- a/euscanwww/runtests.py +++ b/euscanwww/runtests.py @@ -29,6 +29,7 @@ settings.configure( USE_TZ=True, TASKS_CONCURRENTLY=8, TASKS_SUBTASK_PACKAGES=32, + AUTH_PROFILE_MODULE="djeuscan.UserProfile" ) diff --git a/pym/euscan/handlers/__init__.py b/pym/euscan/handlers/__init__.py index 495ef7c..899ef24 100644 --- a/pym/euscan/handlers/__init__.py +++ b/pym/euscan/handlers/__init__.py @@ -1,35 +1,86 @@ -import pkgutil - -# autoimport all modules in this directory and append them to handlers list -handlers = [] -for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): - module = loader.find_module(module_name).load_module(module_name) - handlers.append(module) - -# sort handlers by priority (e.g.: generic should be run lastly) -handlers = sorted( - handlers, - key=lambda handler: handler.PRIORITY, - reverse=True -) +import sys +from euscan import CONFIG, output +from euscan.handlers.package import handlers as pkg_handlers +from euscan.handlers.url import handlers as url_handlers -def find_best_handler(pkg, url): - for handler in handlers: +def find_best_pkg_handler(pkg): + """ + Find the best handler for the given package + """ + for handler in pkg_handlers: + if handler.can_handle(pkg): + return handler + return None + + +def find_best_url_handler(pkg, url): + """ + Find the best handler for the given url + """ + for handler in url_handlers: if handler.can_handle(pkg, url): return handler return None -def scan(pkg, url): - handler = find_best_handler(pkg, url) - if handler: - return handler.scan(pkg, url) - return [] +def scan(pkg, urls, on_progress=None): + """ + Scans upstream for the given package. + First tries if a package wide handler is available, then fallbacks + in url handling. + """ + pkg_handler = find_best_pkg_handler(pkg) + if pkg_handler: + if on_progress: + on_progress(increment=35) + if not CONFIG['quiet'] and not CONFIG['format']: + sys.stdout.write("\n") -def brute_force(pkg, url): - handler = find_best_handler(pkg, url) - if handler: - return handler.brute_force(pkg, url) - return [] + versions = pkg_handler.scan(pkg) + + if on_progress: + on_progress(increment=35) + return versions + + if on_progress: + progress_available = 70 + num_urls = sum([len(urls[fn]) for fn in urls]) + if num_urls > 0: + progress_increment = progress_available / num_urls + else: + progress_increment = 0 + + versions = [] + + for filename in urls: + for url in urls[filename]: + if on_progress and progress_available > 0: + on_progress(increment=progress_increment) + progress_available -= progress_increment + + if not CONFIG['quiet'] and not CONFIG['format']: + sys.stdout.write("\n") + output.einfo("SRC_URI is '%s'" % url) + + if '://' not in url: + output.einfo("Invalid url '%s'" % url) + continue + + try: + url_handler = find_best_url_handler(pkg, url) + versions.extend(url_handler.scan(pkg, url)) + except Exception as e: + output.ewarn( + "Handler failed: [%s] %s" % + (e.__class__.__name__, e.message) + ) + + if versions and CONFIG['oneshot']: + break + + if on_progress and progress_available > 0: + on_progress(increment=progress_available) + + return versions diff --git a/pym/euscan/handlers/package/__init__.py b/pym/euscan/handlers/package/__init__.py new file mode 100644 index 0000000..8530b10 --- /dev/null +++ b/pym/euscan/handlers/package/__init__.py @@ -0,0 +1,19 @@ +""" +Package wide handlers for scanning upstream +""" + +import pkgutil + +handlers = [] + +# autoimport all modules in this directory and append them to handlers list +for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): + module = loader.find_module(module_name).load_module(module_name) + handlers.append(module) + +# sort handlers by priority +handlers = sorted( + handlers, + key=lambda handler: handler.PRIORITY, + reverse=True +) diff --git a/pym/euscan/handlers/package/remote_id.py b/pym/euscan/handlers/package/remote_id.py new file mode 100644 index 0000000..b95f856 --- /dev/null +++ b/pym/euscan/handlers/package/remote_id.py @@ -0,0 +1,43 @@ +from euscan.handlers.url import handlers +from euscan import output + +PRIORITY = 100 + +HANDLER_NAME = "remote_id" +CONFIDENCE = 100.0 + + +url_handlers = {handler.HANDLER_NAME: handler for handler in handlers} + + +def can_handle(pkg): + # Return True if there's at least one remote-id that can be + # handled by euscan + try: + remoteids = pkg.metadata.upstream()[0].upstream_remoteids() + except IndexError: + pass + else: + if len(remoteids) > 0: + for remote_value, remote_type in remoteids: + if remote_type in url_handlers: + return True + return False + + +def scan(pkg): + output.einfo("Using remote-id data") + + ret = [] + + remoteids = pkg.metadata.upstream()[0].upstream_remoteids() + for remote_value, remote_type in remoteids: + if remote_type in url_handlers: + remote_data = remote_value.split("/") + scan_remote = getattr( + url_handlers[remote_type], "scan_remote", None + ) + if scan_remote: + for url, pv in scan_remote(pkg, remote_data): + ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) + return ret diff --git a/pym/euscan/handlers/watch.py b/pym/euscan/handlers/package/watch.py similarity index 94% rename from pym/euscan/handlers/watch.py rename to pym/euscan/handlers/package/watch.py index a129281..14f25d2 100644 --- a/pym/euscan/handlers/watch.py +++ b/pym/euscan/handlers/package/watch.py @@ -3,7 +3,7 @@ import urllib2 import portage -from euscan.handlers import generic +from euscan.handlers.url import generic from euscan import output, helpers PRIORITY = 100 @@ -15,7 +15,7 @@ CONFIDENCE = 100.0 is_pattern = r"\([^\/]+\)" -def can_handle(pkg, url): +def can_handle(pkg): try: return pkg.metadata._xml_tree.find("upstream").find("watch") \ is not None @@ -116,7 +116,7 @@ def handle_directory_patterns(base, file_pattern): for _, path in scan_data] -def scan(pkg, url): +def scan(pkg): output.einfo("Using watch data") cp, ver, rev = portage.pkgsplit(pkg.cpv) @@ -126,18 +126,14 @@ def scan(pkg, url): if not re.search(is_pattern, base): steps = [(base, file_pattern)] res = generic.scan_directory_recursive( - cp, ver, rev, "", steps, url + cp, ver, rev, "", steps, base ) else: res = [] for step in handle_directory_patterns(base, file_pattern): res += generic.scan_directory_recursive( - cp, ver, rev, "", [step], url + cp, ver, rev, "", [step], base ) results += clean_results(res, versionmangle, urlmangle) return results - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/handlers/url/__init__.py b/pym/euscan/handlers/url/__init__.py new file mode 100644 index 0000000..7328644 --- /dev/null +++ b/pym/euscan/handlers/url/__init__.py @@ -0,0 +1,19 @@ +""" +Url wide handlers for scanning upstream +""" + +import pkgutil + +handlers = [] + +# autoimport all modules in this directory and append them to handlers list +for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): + module = loader.find_module(module_name).load_module(module_name) + handlers.append(module) + +# sort handlers by priority +handlers = sorted( + handlers, + key=lambda handler: handler.PRIORITY, + reverse=True +) diff --git a/pym/euscan/handlers/cpan.py b/pym/euscan/handlers/url/cpan.py similarity index 86% rename from pym/euscan/handlers/cpan.py rename to pym/euscan/handlers/url/cpan.py index a54641f..0721324 100644 --- a/pym/euscan/handlers/cpan.py +++ b/pym/euscan/handlers/url/cpan.py @@ -81,12 +81,22 @@ def cpan_vercmp(cp, a, b): def scan(pkg, url): cp, ver, rev = portage.pkgsplit(pkg.cpv) - pkg = guess_package(cp, url) + remote_pkg = guess_package(cp, url) - orig_url = url - url = 'http://search.cpan.org/api/dist/%s' % pkg + output.einfo("Using CPAN API: %s", remote_pkg) - output.einfo("Using: " + url) + result = scan_remote(pkg, [remote_pkg]) + + ret = [] + for url, pv in result: + ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) + return ret + + +def scan_remote(pkg, remote_data): + remote_pkg = remote_data[0] + url = 'http://search.cpan.org/api/dist/%s' % remote_pkg + cp, ver, rev = portage.pkgsplit(pkg.cpv) try: fp = helpers.urlopen(url) @@ -125,13 +135,6 @@ def scan(pkg, url): version['archive'] ) - if url == orig_url: - continue - - ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) + ret.append((url, pv)) return ret - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/url/generic.py similarity index 80% rename from pym/euscan/handlers/generic.py rename to pym/euscan/handlers/url/generic.py index 12c4a99..1f43a7a 100644 --- a/pym/euscan/handlers/generic.py +++ b/pym/euscan/handlers/url/generic.py @@ -108,43 +108,50 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): def scan(pkg, url): - for bu in SCANDIR_BLACKLIST_URLS: - if re.match(bu, url): - output.einfo("%s is blacklisted by rule %s" % (url, bu)) + if CONFIG["scan-dir"]: + for bu in SCANDIR_BLACKLIST_URLS: + if re.match(bu, url): + output.einfo("%s is blacklisted by rule %s" % (url, bu)) + return [] + + resolved_url = helpers.parse_mirror(url) + if not resolved_url: return [] - resolved_url = helpers.parse_mirror(url) - if not resolved_url: - return [] + cp, ver, rev = portage.pkgsplit(pkg.cpv) - cp, ver, rev = portage.pkgsplit(pkg.cpv) + # 'Hack' for _beta/_rc versions where _ is used instead of - + if ver not in resolved_url: + newver = helpers.version_change_end_sep(ver) + if newver and newver in resolved_url: + output.einfo( + "Version: using %s instead of %s" % (newver, ver) + ) + ver = newver - # 'Hack' for _beta/_rc versions where _ is used instead of - - if ver not in resolved_url: - newver = helpers.version_change_end_sep(ver) - if newver and newver in resolved_url: + template = helpers.template_from_url(resolved_url, ver) + if '${' not in template: output.einfo( - "Version: using %s instead of %s" % (newver, ver) + "Url doesn't seems to depend on version: %s not found in %s" % + (ver, resolved_url) ) - ver = newver + return [] + else: + output.einfo("Scanning: %s" % template) - template = helpers.template_from_url(resolved_url, ver) - if '${' not in template: - output.einfo( - "Url doesn't seems to depend on version: %s not found in %s" % - (ver, resolved_url) - ) - return [] - else: - output.einfo("Scanning: %s" % template) + steps = helpers.generate_scan_paths(template) + ret = scan_directory_recursive(cp, ver, rev, "", steps, url) - steps = helpers.generate_scan_paths(template) - ret = scan_directory_recursive(cp, ver, rev, "", steps, url) + if not ret: + ret = brute_force(pkg, url) return ret def brute_force(pkg, url): + if CONFIG["brute-force"] == 0: + return [] + cp, ver, rev = portage.pkgsplit(pkg.cpv) url = helpers.parse_mirror(url) diff --git a/pym/euscan/handlers/github.py b/pym/euscan/handlers/url/github.py similarity index 92% rename from pym/euscan/handlers/github.py rename to pym/euscan/handlers/url/github.py index 9bb5596..e4ebe10 100644 --- a/pym/euscan/handlers/github.py +++ b/pym/euscan/handlers/url/github.py @@ -44,6 +44,7 @@ def scan(pkg, url): (user, project)) dls = json.load(dlreq) + ret = [] for dl in dls: m = fnre.match(dl['name']) @@ -51,8 +52,6 @@ def scan(pkg, url): pv = helpers.gentoo_mangle_version(m.group(1)) if helpers.version_filtered(cp, ver, pv): continue - yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE) - -def brute_force(pkg, url): - return [] + ret.append((dl['html_url'], pv, HANDLER_NAME, CONFIDENCE)) + return ret diff --git a/pym/euscan/handlers/kde.py b/pym/euscan/handlers/url/kde.py similarity index 65% rename from pym/euscan/handlers/kde.py rename to pym/euscan/handlers/url/kde.py index 21722bb..5535158 100644 --- a/pym/euscan/handlers/kde.py +++ b/pym/euscan/handlers/url/kde.py @@ -1,4 +1,4 @@ -from euscan.handlers import generic +from euscan.handlers.url import generic PRIORITY = 90 @@ -29,14 +29,11 @@ def scan(pkg, url): url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/') results += generic.scan(pkg.cpv, url) - return clean_results(results) + if not results: # if nothing was found go brute forcing + results = generic.brute_force(pkg.cpv, url) - -def brute_force(pkg, url): - results = generic.brute_force(pkg.cpv, url) - - if url.startswith('mirror://kde/unstable/'): - url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/') - results += generic.brute_force(pkg.cpv, url) + if url.startswith('mirror://kde/unstable/'): + url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/') + results += generic.brute_force(pkg.cpv, url) return clean_results(results) diff --git a/pym/euscan/handlers/php.py b/pym/euscan/handlers/url/php.py similarity index 97% rename from pym/euscan/handlers/php.py rename to pym/euscan/handlers/url/php.py index 6b74ff6..853059a 100644 --- a/pym/euscan/handlers/php.py +++ b/pym/euscan/handlers/url/php.py @@ -70,7 +70,3 @@ def scan(pkg, url): ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/handlers/pypi.py b/pym/euscan/handlers/url/pypi.py similarity index 86% rename from pym/euscan/handlers/pypi.py rename to pym/euscan/handlers/url/pypi.py index 9cd1620..82251e6 100644 --- a/pym/euscan/handlers/pypi.py +++ b/pym/euscan/handlers/url/pypi.py @@ -29,6 +29,15 @@ def scan(pkg, url): package = guess_package(pkg.cpv, url) + ret = [] + for urls, pv in scan_remote(pkg, [package]): + ret.append((urls, pv, HANDLER_NAME, CONFIDENCE)) + return ret + + +def scan_remote(pkg, remote_data): + package = remote_data[0] + output.einfo("Using PyPi XMLRPC: " + package) client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi') @@ -42,17 +51,11 @@ def scan(pkg, url): cp, ver, rev = portage.pkgsplit(pkg.cpv) ret = [] - for up_pv in versions: pv = helpers.gentoo_mangle_version(up_pv) if helpers.version_filtered(cp, ver, pv): continue urls = client.release_urls(package, up_pv) urls = " ".join([infos['url'] for infos in urls]) - ret.append((urls, pv, HANDLER_NAME, CONFIDENCE)) - + ret.append((urls, pv)) return ret - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/handlers/rubygem.py b/pym/euscan/handlers/url/rubygems.py similarity index 82% rename from pym/euscan/handlers/rubygem.py rename to pym/euscan/handlers/url/rubygems.py index 39e2334..a3021f0 100644 --- a/pym/euscan/handlers/rubygem.py +++ b/pym/euscan/handlers/url/rubygems.py @@ -5,7 +5,7 @@ import urllib2 from euscan import helpers, output -HANDLER_NAME = "rubygem" +HANDLER_NAME = "rubygems" CONFIDENCE = 100.0 PRIORITY = 90 @@ -33,14 +33,23 @@ def scan(pkg, url): 'http://guides.rubygems.org/rubygems-org-api/#gemversion' gem = guess_gem(pkg.cpv, url) + if not gem: output.eerror("Can't guess gem name using %s and %s" % \ (pkg.cpv, url)) return [] - url = 'http://rubygems.org/api/v1/versions/%s.json' % gem + output.einfo("Using RubyGem API: %s" % gem) - output.einfo("Using: " + url) + ret = [] + for url, pv in scan_remote(pkg, [gem]): + ret.append(url, pv, HANDLER_NAME, CONFIDENCE) + return ret + + +def scan_remote(pkg, remote_data): + gem = remote_data[0] + url = 'http://rubygems.org/api/v1/versions/%s.json' % gem try: fp = helpers.urlopen(url) @@ -55,23 +64,14 @@ def scan(pkg, url): data = fp.read() versions = json.loads(data) - if not versions: - return [] - cp, ver, rev = portage.pkgsplit(pkg.cpv) ret = [] - for version in versions: up_pv = version['number'] pv = helpers.gentoo_mangle_version(up_pv) if helpers.version_filtered(cp, ver, pv): continue url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv) - ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) - + ret.append((url, pv)) return ret - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py index 114c81e..5d9fd06 100644 --- a/pym/euscan/scan.py +++ b/pym/euscan/scan.py @@ -44,60 +44,6 @@ def filter_versions(cp, versions): ] -def scan_upstream_urls(pkg, urls, on_progress): - versions = [] - - if on_progress: - progress_available = 70 - num_urls = sum([len(urls[fn]) for fn in urls]) - if num_urls > 0: - progress_increment = progress_available / num_urls - else: - progress_increment = 0 - - for filename in urls: - for url in urls[filename]: - - if on_progress and progress_available > 0: - on_progress(increment=progress_increment) - progress_available -= progress_increment - - if not CONFIG['quiet'] and not CONFIG['format']: - pp.uprint() - output.einfo("SRC_URI is '%s'" % url) - - if '://' not in url: - output.einfo("Invalid url '%s'" % url) - continue - - # Try normal scan - if CONFIG["scan-dir"]: - try: - versions.extend(handlers.scan(pkg, url)) - except Exception as e: - output.ewarn("Handler failed: [%s] %s" - % (e.__class__.__name__, e.message)) - - if versions and CONFIG['oneshot']: - break - - # Brute Force - if CONFIG["brute-force"] > 0: - versions.extend(handlers.brute_force(pkg, url)) - - if versions and CONFIG['oneshot']: - break - - cp, ver, rev = portage.pkgsplit(pkg.cpv) - - result = filter_versions(cp, versions) - - if on_progress and progress_available > 0: - on_progress(increment=progress_available) - - return result - - # gentoolkit stores PORTDB, so even if we modify it to add an overlay # it will still use the old dbapi def reload_gentoolkit(): @@ -120,7 +66,6 @@ def scan_upstream(query, on_progress=None): """ Scans the upstream searching new versions for the given query """ - matches = [] if query.endswith(".ebuild"): @@ -188,10 +133,6 @@ def scan_upstream(query, on_progress=None): output.metadata("description", pkg.environment("DESCRIPTION")) cpv = pkg.cpv - - _, _, ver, _ = portage.catpkgsplit(cpv) - is_current_version_stable = is_version_stable(ver) - metadata = { "EAPI": portage.settings["EAPI"], "SRC_URI": pkg.environment("SRC_URI", False), @@ -212,15 +153,20 @@ def scan_upstream(query, on_progress=None): else: urls = alist - # output scan time for formatted output - scan_time = (datetime.now() - start_time).total_seconds() - output.metadata("scan_time", scan_time, show=False) + versions = handlers.scan(pkg, urls, on_progress) - result = scan_upstream_urls(pkg, urls, on_progress) + cp, ver, rev = portage.pkgsplit(pkg.cpv) + + result = filter_versions(cp, versions) if on_progress: on_progress(increment=10) + # output scan time for formatted output + scan_time = (datetime.now() - start_time).total_seconds() + output.metadata("scan_time", scan_time, show=False) + + is_current_version_stable = is_version_stable(ver) if len(result) > 0: if not (CONFIG['format'] or CONFIG['quiet']): print("\n", file=sys.stderr)