From 5add4ddfcc77ff2e6c6c9299a58aa231fd14418c Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Sat, 21 Jul 2012 07:36:06 +0200 Subject: [PATCH 01/17] djeuscan: helpers.py is actually unused Signed-off-by: Corentin Chary --- euscanwww/djeuscan/helpers.py | 108 ---------------------------------- 1 file changed, 108 deletions(-) delete mode 100644 euscanwww/djeuscan/helpers.py diff --git a/euscanwww/djeuscan/helpers.py b/euscanwww/djeuscan/helpers.py deleted file mode 100644 index f7a6b82..0000000 --- a/euscanwww/djeuscan/helpers.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -djeuscan.helpers -""" - -from distutils.version import StrictVersion, LooseVersion - - -def xint(i): - """ - Tries to cast to int, fallbacks to 0 - """ - try: - return int(i) - except Exception: - return 0 - - -def select_related_last_versions(queryset): - queryset = queryset.select_related( - 'last_version_gentoo', - 'last_version_overlay', - 'last_version_upstream' - ) - - -def version_key(version): - version = version.version - try: - return StrictVersion(version) - # in case of abnormal version number, fall back to LooseVersion - except ValueError: - return LooseVersion(version) - - -def packages_from_names(data): - """ - Returns a list of Package objects from a string of names - """ - - from djeuscan.models import Package - - packages = [] - data = data.replace("\r", "") - - for pkg in data.split('\n'): - if '/' in pkg: - cat, pkg = pkg.split('/') - packages.extend(Package.objects.filter(category=cat, name=pkg)) - else: - packages.extend(Package.objects.filter(name=pkg)) - return packages - - -def rename_fields(vqs, fields): - ret = [] - for n in vqs: - for tr in fields: - if tr[0] in n: - n[tr[1]] = n[tr[0]] - del n[tr[0]] - ret.append(n) - return ret - - -class catch_and_return(object): - def __init__(self, err, response): - self.err = err - self.response = response - - def __call__(self, fn): - def wrapper(*args, **kwargs): - try: - return fn(*args, **kwargs) - except self.err: - return self.response - return wrapper - - -def get_account_categories(user): - from djeuscan.models import Package, CategoryAssociation - - # TODO: This is quite ugly - category_names = [obj.category for obj in - CategoryAssociation.objects.filter(user=user)] - return [c for c in Package.objects.categories() - if c["category"] in category_names] - - -def get_account_herds(user): - from djeuscan.models import Package, HerdAssociation - - ids = [obj.herd.pk for obj in - HerdAssociation.objects.filter(user=user)] - return Package.objects.herds(ids=ids) - - -def get_account_maintainers(user): - from djeuscan.models import Package, MaintainerAssociation - - ids = [obj.maintainer.pk for obj in - MaintainerAssociation.objects.filter(user=user)] - return Package.objects.maintainers(ids=ids) - - -def get_account_packages(user): - from djeuscan.models import PackageAssociation - return [obj.package for obj in - PackageAssociation.objects.filter(user=user)] From 65618e3056e843361ed420577283435bbacccfdb Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Sat, 21 Jul 2012 07:36:35 +0200 Subject: [PATCH 02/17] djeuscan: portage. can be used instead of PORTDB Signed-off-by: Corentin Chary --- euscanwww/djeuscan/processing/scan/scan_portage.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/euscanwww/djeuscan/processing/scan/scan_portage.py b/euscanwww/djeuscan/processing/scan/scan_portage.py index a30254e..79fa62a 100644 --- a/euscanwww/djeuscan/processing/scan/scan_portage.py +++ b/euscanwww/djeuscan/processing/scan/scan_portage.py @@ -13,9 +13,6 @@ from euscan.version import get_version_type from djeuscan.processing import FakeLogger from djeuscan.models import Package, Version, VersionLog -PORTDB = None - - class ScanPortage(object): def __init__(self, logger=None, no_log=False, purge_packages=False, purge_versions=False): @@ -24,10 +21,6 @@ class ScanPortage(object): self.purge_packages = purge_packages self.purge_versions = purge_versions - global PORTDB - if not PORTDB: # Lazy loading for portdb - PORTDB = portage.db[portage.root]["porttree"].dbapi - self.style = color_style() self._cache = {'packages': {}, 'versions': {}} @@ -231,7 +224,7 @@ class ScanPortage(object): package.category, package.name, ver, rev, slot, overlay ) - overlay_path = overlay_path or PORTDB.settings["PORTDIR"] + overlay_path = overlay_path or portage.settings["PORTDIR"] package_path = join(overlay_path, package.category, package.name) ebuild_path = join(package_path, "%s.ebuild" % cpv.split("/")[-1]) metadata_path = join(package_path, "metadata.xml") From 634c2db38950910e8e99d10385899364b5042713 Mon Sep 17 00:00:00 2001 From: volpino Date: Sun, 22 Jul 2012 10:26:40 +0200 Subject: [PATCH 03/17] euscan: euscan_patch_metadata --diff option Signed-off-by: volpino --- bin/euscan_patch_metadata | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/bin/euscan_patch_metadata b/bin/euscan_patch_metadata index 439105e..b5649ed 100755 --- a/bin/euscan_patch_metadata +++ b/bin/euscan_patch_metadata @@ -1,6 +1,7 @@ #!/usr/bin/env python import os +import sys import re import urllib from tempfile import mkstemp @@ -8,6 +9,7 @@ import tarfile import gzip import logging import shutil +import subprocess from gentoolkit.query import Query from BeautifulSoup import BeautifulSoup, SoupStrainer @@ -130,7 +132,7 @@ def get_deb_url(name): return deb_url, deb_type -def patch_metadata(metadata_path, watch_data): +def patch_metadata(metadata_path, watch_data, diff=False): watch_data = "\n".join([line for line in watch_data.split("\n") if not line.startswith("#")]) # comments watch_data = watch_data.replace("\\\n", "") # remove backslashes @@ -167,10 +169,20 @@ def patch_metadata(metadata_path, watch_data): (rindent, watch_tag, rindent) data = data.replace('', rep, 1) - print data + if not diff: + return data + else: + _, data_path = mkstemp() + with open(data_path, "w") as f: + f.write(data) + sub = subprocess.Popen(["diff", metadata_path, data_path], + stdout=subprocess.PIPE) + content = sub.stdout.read() + os.unlink(data_path) + return content -def process_package(query): +def process_package(query, diff=False): matches = Query(query).smart_find( in_installed=True, in_porttree=True, @@ -194,7 +206,7 @@ def process_package(query): if watch_data is None: logger.error(" No watch file found") else: - patch_metadata(metadata_path, watch_data) + return patch_metadata(metadata_path, watch_data, diff=diff) def main(): @@ -202,13 +214,17 @@ def main(): p = optparse.OptionParser( usage="usage: %prog [ [...]]", ) + p.add_option('-d', '--diff', action="store_true", dest="diff", + default=False, + help="Outputs a diff") opts, packages = p.parse_args() - logging.basicConfig(level=logging.INFO, format='%(message)s') + logging.basicConfig(stream=sys.stderr, level=logging.INFO, + format='%(message)s') for package in packages: logger.info("Processing %s..." % package) - process_package(package) + print process_package(package, opts.diff) if __name__ == "__main__": main() From 8f768bd4fd7807e843b23ce0cc89a96798e382d2 Mon Sep 17 00:00:00 2001 From: volpino Date: Sun, 22 Jul 2012 10:34:13 +0200 Subject: [PATCH 04/17] euscanwww: From fixture to south data migration Signed-off-by: volpino --- euscanwww/djeuscan/fixtures/initial_data.json | 89 ----- .../0015_initial_celery_periodictasks.py | 304 ++++++++++++++++++ 2 files changed, 304 insertions(+), 89 deletions(-) delete mode 100644 euscanwww/djeuscan/fixtures/initial_data.json create mode 100644 euscanwww/djeuscan/migrations/0015_initial_celery_periodictasks.py diff --git a/euscanwww/djeuscan/fixtures/initial_data.json b/euscanwww/djeuscan/fixtures/initial_data.json deleted file mode 100644 index 250ab2f..0000000 --- a/euscanwww/djeuscan/fixtures/initial_data.json +++ /dev/null @@ -1,89 +0,0 @@ -[ - { - "pk": 1, - "model": "djcelery.intervalschedule", - "fields": { - "every": 1, - "period": "minutes" - } - }, - { - "pk": 2, - "model": "djcelery.intervalschedule", - "fields": { - "every": 1, - "period": "days" - } - }, - { - "pk": 3, - "model": "djcelery.intervalschedule", - "fields": { - "every": 7, - "period": "days" - } - }, - { - "pk": 2, - "model": "djcelery.periodictask", - "fields": { - "task": "djeuscan.tasks.consume_refresh_package_request", - "name": "User refresh queries", - "exchange": null, - "last_run_at": null, - "args": "[]", - "enabled": true, - "routing_key": null, - "crontab": null, - "interval": 1, - "queue": null, - "total_run_count": 0, - "expires": null, - "kwargs": "{}", - "date_changed": "2012-07-21T07:55:54.716Z", - "description": "" - } - }, - { - "pk": 3, - "model": "djcelery.periodictask", - "fields": { - "task": "djeuscan.tasks.update_portage", - "name": "Daily portage scan", - "exchange": null, - "last_run_at": null, - "args": "[]", - "enabled": true, - "routing_key": null, - "crontab": null, - "interval": 2, - "queue": null, - "total_run_count": 0, - "expires": null, - "kwargs": "{}", - "date_changed": "2012-07-21T07:56:25.235Z", - "description": "" - } - }, - { - "pk": 4, - "model": "djcelery.periodictask", - "fields": { - "task": "djeuscan.tasks.update_upstream", - "name": "Weekly upstream update", - "exchange": null, - "last_run_at": null, - "args": "[]", - "enabled": true, - "routing_key": null, - "crontab": null, - "interval": 3, - "queue": null, - "total_run_count": 0, - "expires": null, - "kwargs": "{}", - "date_changed": "2012-07-21T07:56:53.470Z", - "description": "" - } - } -] diff --git a/euscanwww/djeuscan/migrations/0015_initial_celery_periodictasks.py b/euscanwww/djeuscan/migrations/0015_initial_celery_periodictasks.py new file mode 100644 index 0000000..a5b3bed --- /dev/null +++ b/euscanwww/djeuscan/migrations/0015_initial_celery_periodictasks.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- +from south.v2 import DataMigration + + +class Migration(DataMigration): + + def forwards(self, orm): + every_minute = orm["djcelery.IntervalSchedule"].objects.create( + every=1, period="minutes" + ) + every_day = orm["djcelery.IntervalSchedule"].objects.create( + every=1, period="days" + ) + every_week = orm["djcelery.IntervalSchedule"].objects.create( + every=7, period="days" + ) + orm["djcelery.PeriodicTask"].objects.create( + name="Refresh package queries", + task="djeuscan.tasks.consume_refresh_package_request", + interval=every_minute + ) + orm["djcelery.PeriodicTask"].objects.create( + name="Daily portage update", + task="djeuscan.tasks.update_portage", + interval=every_day + ) + orm["djcelery.PeriodicTask"].objects.create( + name="Weekly upstream update", + task="djeuscan.tasks.update_upstream", + interval=every_week + ) + + def backwards(self, orm): + orm["djcelery.IntervalSchedule"].objects.all().delete() + orm["djcelery.PeriodicTask"].objects.all().delete() + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'djcelery.crontabschedule': { + 'Meta': {'object_name': 'CrontabSchedule'}, + 'day_of_month': ('django.db.models.fields.CharField', [], {'default': "'*'", 'max_length': '64'}), + 'day_of_week': ('django.db.models.fields.CharField', [], {'default': "'*'", 'max_length': '64'}), + 'hour': ('django.db.models.fields.CharField', [], {'default': "'*'", 'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'minute': ('django.db.models.fields.CharField', [], {'default': "'*'", 'max_length': '64'}), + 'month_of_year': ('django.db.models.fields.CharField', [], {'default': "'*'", 'max_length': '64'}) + }, + 'djcelery.intervalschedule': { + 'Meta': {'object_name': 'IntervalSchedule'}, + 'every': ('django.db.models.fields.IntegerField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'period': ('django.db.models.fields.CharField', [], {'max_length': '24'}) + }, + 'djcelery.periodictask': { + 'Meta': {'object_name': 'PeriodicTask'}, + 'args': ('django.db.models.fields.TextField', [], {'default': "'[]'", 'blank': 'True'}), + 'crontab': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djcelery.CrontabSchedule']", 'null': 'True', 'blank': 'True'}), + 'date_changed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'exchange': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'expires': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djcelery.IntervalSchedule']", 'null': 'True', 'blank': 'True'}), + 'kwargs': ('django.db.models.fields.TextField', [], {'default': "'{}'", 'blank': 'True'}), + 'last_run_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'}), + 'queue': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'routing_key': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'task': ('django.db.models.fields.CharField', [], {'max_length': '200'}), + 'total_run_count': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}) + }, + 'djcelery.periodictasks': { + 'Meta': {'object_name': 'PeriodicTasks'}, + 'ident': ('django.db.models.fields.SmallIntegerField', [], {'default': '1', 'unique': 'True', 'primary_key': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {}) + }, + 'djcelery.taskmeta': { + 'Meta': {'object_name': 'TaskMeta', 'db_table': "'celery_taskmeta'"}, + 'date_done': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'meta': ('djcelery.picklefield.PickledObjectField', [], {'default': 'None', 'null': 'True'}), + 'result': ('djcelery.picklefield.PickledObjectField', [], {'default': 'None', 'null': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'PENDING'", 'max_length': '50'}), + 'task_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'traceback': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'djcelery.tasksetmeta': { + 'Meta': {'object_name': 'TaskSetMeta', 'db_table': "'celery_tasksetmeta'"}, + 'date_done': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'result': ('djcelery.picklefield.PickledObjectField', [], {}), + 'taskset_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'djcelery.taskstate': { + 'Meta': {'ordering': "['-tstamp']", 'object_name': 'TaskState'}, + 'args': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'eta': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'expires': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'kwargs': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'db_index': 'True'}), + 'result': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'retries': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'runtime': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'state': ('django.db.models.fields.CharField', [], {'max_length': '64', 'db_index': 'True'}), + 'task_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '36'}), + 'traceback': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'tstamp': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'worker': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djcelery.WorkerState']", 'null': 'True'}) + }, + 'djcelery.workerstate': { + 'Meta': {'ordering': "['-last_heartbeat']", 'object_name': 'WorkerState'}, + 'hostname': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_heartbeat': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}) + }, + 'djeuscan.categoryassociation': { + 'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.categorylog': { + 'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.euscanresult': { + 'Meta': {'object_name': 'EuscanResult'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}) + }, + 'djeuscan.herd': { + 'Meta': {'object_name': 'Herd'}, + 'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'}) + }, + 'djeuscan.herdassociation': { + 'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.herdlog': { + 'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.log': { + 'Meta': {'object_name': 'Log'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'djeuscan.maintainer': { + 'Meta': {'object_name': 'Maintainer'}, + 'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.maintainerassociation': { + 'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.maintainerlog': { + 'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}) + }, + 'djeuscan.overlayassociation': { + 'Meta': {'unique_together': "(['user', 'overlay'],)", 'object_name': 'OverlayAssociation'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.package': { + 'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}), + 'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}), + 'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.packageassociation': { + 'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.problemreport': { + 'Meta': {'object_name': 'ProblemReport'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Version']", 'null': 'True', 'blank': 'True'}) + }, + 'djeuscan.refreshpackagequery': { + 'Meta': {'object_name': 'RefreshPackageQuery'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'symmetrical': 'False'}) + }, + 'djeuscan.version': { + 'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'}, + 'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'ebuild_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'metadata_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.versionlog': { + 'Meta': {'object_name': 'VersionLog'}, + 'action': ('django.db.models.fields.IntegerField', [], {}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.worldlog': { + 'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + } + } + + complete_apps = ['djcelery', 'djeuscan'] + symmetrical = True From 2b09bfba8ace32b9e905bf001ec6bc7e6f82b149 Mon Sep 17 00:00:00 2001 From: volpino Date: Tue, 24 Jul 2012 08:10:56 +0200 Subject: [PATCH 05/17] euscan: Progressbar on stderr Signed-off-by: volpino --- bin/euscan | 3 ++- euscanwww/djeuscan/processing/scan/scan_portage.py | 1 + euscanwww/djeuscan/tests/__init__.py | 1 - pym/euscan/out.py | 2 ++ 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/euscan b/bin/euscan index 8696848..4469f4e 100755 --- a/bin/euscan +++ b/bin/euscan @@ -36,6 +36,7 @@ from euscan import CONFIG, output # Globals isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() +isatty_stderr = os.environ.get('TERM') != 'dumb' and sys.stderr.isatty() def exit_helper(status): @@ -184,7 +185,7 @@ def parse_args(): CONFIG['nocolor'] = True pp.output.nocolor() elif o in ("-p", "--progress"): - CONFIG['progress'] = isatty + CONFIG['progress'] = isatty_stderr elif o in ("--mirror"): CONFIG['mirror'] = True elif o in ("-i", "--ignore-pre-release"): diff --git a/euscanwww/djeuscan/processing/scan/scan_portage.py b/euscanwww/djeuscan/processing/scan/scan_portage.py index 79fa62a..f4ca15d 100644 --- a/euscanwww/djeuscan/processing/scan/scan_portage.py +++ b/euscanwww/djeuscan/processing/scan/scan_portage.py @@ -13,6 +13,7 @@ from euscan.version import get_version_type from djeuscan.processing import FakeLogger from djeuscan.models import Package, Version, VersionLog + class ScanPortage(object): def __init__(self, logger=None, no_log=False, purge_packages=False, purge_versions=False): diff --git a/euscanwww/djeuscan/tests/__init__.py b/euscanwww/djeuscan/tests/__init__.py index 5e37ff2..78a1082 100644 --- a/euscanwww/djeuscan/tests/__init__.py +++ b/euscanwww/djeuscan/tests/__init__.py @@ -3,7 +3,6 @@ from urllib import urlencode from django.test import TestCase from django.test.client import Client from django.core.urlresolvers import reverse -from django.contrib.auth.models import User from djeuscan.tests.euscan_factory import UserFactory diff --git a/pym/euscan/out.py b/pym/euscan/out.py index c0ed61b..8a505fc 100644 --- a/pym/euscan/out.py +++ b/pym/euscan/out.py @@ -1,3 +1,4 @@ +import sys from io import StringIO from collections import defaultdict import json @@ -37,6 +38,7 @@ class ProgressHandler(object): def progress_bar(): on_progress = None progress_bar = TermProgressBar(title="euscan") + progress_bar.file = sys.stderr progress_handler = ProgressHandler(progress_bar) on_progress = progress_handler.on_progress From 8e08b6a33bb1c0771fe0534c0e070f46f50d96b8 Mon Sep 17 00:00:00 2001 From: volpino Date: Tue, 24 Jul 2012 08:56:00 +0200 Subject: [PATCH 06/17] euscan: handlers now use a Package object instead of cpv Signed-off-by: volpino --- pym/euscan/handlers/__init__.py | 16 ++++++++-------- pym/euscan/handlers/cpan.py | 8 ++++---- pym/euscan/handlers/generic.py | 10 +++++----- pym/euscan/handlers/github.py | 10 +++++----- pym/euscan/handlers/kde.py | 14 +++++++------- pym/euscan/handlers/php.py | 14 +++++++------- pym/euscan/handlers/pypi.py | 10 +++++----- pym/euscan/handlers/rubygem.py | 12 ++++++------ pym/euscan/scan.py | 10 +++++----- 9 files changed, 52 insertions(+), 52 deletions(-) diff --git a/pym/euscan/handlers/__init__.py b/pym/euscan/handlers/__init__.py index bcbd680..495ef7c 100644 --- a/pym/euscan/handlers/__init__.py +++ b/pym/euscan/handlers/__init__.py @@ -14,22 +14,22 @@ handlers = sorted( ) -def find_best_handler(cpv, url): +def find_best_handler(pkg, url): for handler in handlers: - if handler.can_handle(cpv, url): + if handler.can_handle(pkg, url): return handler return None -def scan(cpv, url): - handler = find_best_handler(cpv, url) +def scan(pkg, url): + handler = find_best_handler(pkg, url) if handler: - return handler.scan(cpv, url) + return handler.scan(pkg, url) return [] -def brute_force(cpv, url): - handler = find_best_handler(cpv, url) +def brute_force(pkg, url): + handler = find_best_handler(pkg, url) if handler: - return handler.brute_force(cpv, url) + return handler.brute_force(pkg, url) return [] diff --git a/pym/euscan/handlers/cpan.py b/pym/euscan/handlers/cpan.py index 7115c20..091c64c 100644 --- a/pym/euscan/handlers/cpan.py +++ b/pym/euscan/handlers/cpan.py @@ -12,7 +12,7 @@ PRIORITY = 100 _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*") -def can_handle(cpv, url): +def can_handle(pkg, url): return url.startswith('mirror://cpan/') @@ -79,8 +79,8 @@ def cpan_vercmp(cp, a, b): return 1 -def scan(cpv, url): - cp, ver, rev = portage.pkgsplit(cpv) +def scan(pkg, url): + cp, ver, rev = portage.pkgsplit(pkg.cpv) pkg = guess_package(cp, url) orig_url = url @@ -133,5 +133,5 @@ def scan(cpv, url): return ret -def brute_force(cpv, url): +def brute_force(pkg, url): return [] diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/generic.py index 5c6d467..a9a3048 100644 --- a/pym/euscan/handlers/generic.py +++ b/pym/euscan/handlers/generic.py @@ -104,7 +104,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): return versions -def scan(cpv, url): +def scan(pkg, url): for bu in SCANDIR_BLACKLIST_URLS: if re.match(bu, url): output.einfo("%s is blacklisted by rule %s" % (url, bu)) @@ -114,7 +114,7 @@ def scan(cpv, url): if not resolved_url: return [] - cp, ver, rev = portage.pkgsplit(cpv) + cp, ver, rev = portage.pkgsplit(pkg.cpv) # 'Hack' for _beta/_rc versions where _ is used instead of - if ver not in resolved_url: @@ -141,8 +141,8 @@ def scan(cpv, url): return ret -def brute_force(cpv, url): - cp, ver, rev = portage.pkgsplit(cpv) +def brute_force(pkg, url): + cp, ver, rev = portage.pkgsplit(pkg.cpv) url = helpers.parse_mirror(url) if not url: @@ -226,5 +226,5 @@ def brute_force(cpv, url): return result -def can_handle(cpv, url): +def can_handle(pkg, url): return True diff --git a/pym/euscan/handlers/github.py b/pym/euscan/handlers/github.py index ac5baca..76c50c9 100644 --- a/pym/euscan/handlers/github.py +++ b/pym/euscan/handlers/github.py @@ -11,7 +11,7 @@ CONFIDENCE = 100.0 PRIORITY = 100 -def can_handle(cpv, url): +def can_handle(pkg, url): return url.startswith('mirror://github/') @@ -22,13 +22,13 @@ def guess_package(cp, url): return (match.group(1), match.group(2), match.group(3)) -def scan(cpv, url): +def scan(pkg, url): 'http://developer.github.com/v3/repos/downloads/' - user, project, filename = guess_package(cpv, url) + user, project, filename = guess_package(pkg.cpv, url) # find out where version is expected to be found - cp, ver, rev = portage.pkgsplit(cpv) + cp, ver, rev = portage.pkgsplit(pkg.cpv) if ver not in filename: return @@ -54,5 +54,5 @@ def scan(cpv, url): yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE) -def brute_force(cpv, url): +def brute_force(pkg, url): return [] diff --git a/pym/euscan/handlers/kde.py b/pym/euscan/handlers/kde.py index 406b079..1dcead6 100644 --- a/pym/euscan/handlers/kde.py +++ b/pym/euscan/handlers/kde.py @@ -5,7 +5,7 @@ PRIORITY = 100 HANDLER_NAME = "kde" -def can_handle(cpv, url): +def can_handle(pkg, url): if url.startswith('mirror://kde/'): return True return False @@ -22,21 +22,21 @@ def clean_results(results): return ret -def scan(cpv, url): - results = generic.scan(cpv, url) +def scan(pkg, url): + results = generic.scan(pkg.cpv, url) if url.startswith('mirror://kde/unstable/'): url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/') - results += generic.scan(cpv, url) + results += generic.scan(pkg.cpv, url) return clean_results(results) -def brute_force(cpv, url): - results = generic.brute_force(cpv, url) +def brute_force(pkg, url): + results = generic.brute_force(pkg.cpv, url) if url.startswith('mirror://kde/unstable/'): url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/') - results += generic.brute_force(cpv, url) + results += generic.brute_force(pkg.cpv, url) return clean_results(results) diff --git a/pym/euscan/handlers/php.py b/pym/euscan/handlers/php.py index 158280c..36bf3d7 100644 --- a/pym/euscan/handlers/php.py +++ b/pym/euscan/handlers/php.py @@ -10,7 +10,7 @@ CONFIDENCE = 100.0 PRIORITY = 100 -def can_handle(cpv, url): +def can_handle(pkg, url): if url.startswith('http://pear.php.net/get/'): return True if url.startswith('http://pecl.php.net/get/'): @@ -30,12 +30,12 @@ def guess_package_and_channel(cp, url): return pkg, host -def scan(cpv, url): - cp, ver, rev = portage.pkgsplit(cpv) - pkg, channel = guess_package_and_channel(cp, url) +def scan(pkg, url): + cp, ver, rev = portage.pkgsplit(pkg.cpv) + package, channel = guess_package_and_channel(cp, url) orig_url = url - url = 'http://%s/rest/r/%s/allreleases.xml' % (channel, pkg.lower()) + url = 'http://%s/rest/r/%s/allreleases.xml' % (channel, package.lower()) output.einfo("Using: " + url) @@ -62,7 +62,7 @@ def scan(cpv, url): if helpers.version_filtered(cp, ver, pv): continue - url = 'http://%s/get/%s-%s.tgz' % (channel, pkg, up_pv) + url = 'http://%s/get/%s-%s.tgz' % (channel, package, up_pv) if url == orig_url: continue @@ -72,5 +72,5 @@ def scan(cpv, url): return ret -def brute_force(cpv, url): +def brute_force(pkg, url): return [] diff --git a/pym/euscan/handlers/pypi.py b/pym/euscan/handlers/pypi.py index 1dcad99..74aaeb3 100644 --- a/pym/euscan/handlers/pypi.py +++ b/pym/euscan/handlers/pypi.py @@ -10,7 +10,7 @@ CONFIDENCE = 100.0 PRIORITY = 100 -def can_handle(cpv, url): +def can_handle(pkg, url): return url.startswith('mirror://pypi/') @@ -24,10 +24,10 @@ def guess_package(cp, url): return pkg -def scan(cpv, url): +def scan(pkg, url): 'http://wiki.python.org/moin/PyPiXmlRpc' - package = guess_package(cpv, url) + package = guess_package(pkg.cpv, url) output.einfo("Using PyPi XMLRPC: " + package) @@ -39,7 +39,7 @@ def scan(cpv, url): versions.reverse() - cp, ver, rev = portage.pkgsplit(cpv) + cp, ver, rev = portage.pkgsplit(pkg.cpv) ret = [] @@ -54,5 +54,5 @@ def scan(cpv, url): return ret -def brute_force(cpv, url): +def brute_force(pkg, url): return [] diff --git a/pym/euscan/handlers/rubygem.py b/pym/euscan/handlers/rubygem.py index c0f7ade..529e6d4 100644 --- a/pym/euscan/handlers/rubygem.py +++ b/pym/euscan/handlers/rubygem.py @@ -10,7 +10,7 @@ CONFIDENCE = 100.0 PRIORITY = 100 -def can_handle(cpv, url): +def can_handle(pkg, url): return url.startswith('mirror://rubygems/') @@ -29,13 +29,13 @@ def guess_gem(cpv, url): return pkg -def scan(cpv, url): +def scan(pkg, url): 'http://guides.rubygems.org/rubygems-org-api/#gemversion' - gem = guess_gem(cpv, url) + gem = guess_gem(pkg.cpv, url) if not gem: output.eerror("Can't guess gem name using %s and %s" % \ - (cpv, url)) + (pkg.cpv, url)) return [] url = 'http://rubygems.org/api/v1/versions/%s.json' % gem @@ -58,7 +58,7 @@ def scan(cpv, url): if not versions: return [] - cp, ver, rev = portage.pkgsplit(cpv) + cp, ver, rev = portage.pkgsplit(pkg.cpv) ret = [] @@ -73,5 +73,5 @@ def scan(cpv, url): return ret -def brute_force(cpv, url): +def brute_force(pkg, url): return [] diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py index d63488b..114c81e 100644 --- a/pym/euscan/scan.py +++ b/pym/euscan/scan.py @@ -44,7 +44,7 @@ def filter_versions(cp, versions): ] -def scan_upstream_urls(cpv, urls, on_progress): +def scan_upstream_urls(pkg, urls, on_progress): versions = [] if on_progress: @@ -73,7 +73,7 @@ def scan_upstream_urls(cpv, urls, on_progress): # Try normal scan if CONFIG["scan-dir"]: try: - versions.extend(handlers.scan(cpv, url)) + versions.extend(handlers.scan(pkg, url)) except Exception as e: output.ewarn("Handler failed: [%s] %s" % (e.__class__.__name__, e.message)) @@ -83,12 +83,12 @@ def scan_upstream_urls(cpv, urls, on_progress): # Brute Force if CONFIG["brute-force"] > 0: - versions.extend(handlers.brute_force(cpv, url)) + versions.extend(handlers.brute_force(pkg, url)) if versions and CONFIG['oneshot']: break - cp, ver, rev = portage.pkgsplit(cpv) + cp, ver, rev = portage.pkgsplit(pkg.cpv) result = filter_versions(cp, versions) @@ -216,7 +216,7 @@ def scan_upstream(query, on_progress=None): scan_time = (datetime.now() - start_time).total_seconds() output.metadata("scan_time", scan_time, show=False) - result = scan_upstream_urls(pkg.cpv, urls, on_progress) + result = scan_upstream_urls(pkg, urls, on_progress) if on_progress: on_progress(increment=10) From cc7c3786483b849fbd62f3f5c74290a6096bab57 Mon Sep 17 00:00:00 2001 From: volpino Date: Tue, 24 Jul 2012 09:02:01 +0200 Subject: [PATCH 07/17] Revert "djeuscan: helpers.py is actually unused" This reverts commit be9d28c2557bdf09203bbc124b9fd434325ccc47. --- euscanwww/djeuscan/helpers.py | 108 ++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 euscanwww/djeuscan/helpers.py diff --git a/euscanwww/djeuscan/helpers.py b/euscanwww/djeuscan/helpers.py new file mode 100644 index 0000000..f7a6b82 --- /dev/null +++ b/euscanwww/djeuscan/helpers.py @@ -0,0 +1,108 @@ +""" +djeuscan.helpers +""" + +from distutils.version import StrictVersion, LooseVersion + + +def xint(i): + """ + Tries to cast to int, fallbacks to 0 + """ + try: + return int(i) + except Exception: + return 0 + + +def select_related_last_versions(queryset): + queryset = queryset.select_related( + 'last_version_gentoo', + 'last_version_overlay', + 'last_version_upstream' + ) + + +def version_key(version): + version = version.version + try: + return StrictVersion(version) + # in case of abnormal version number, fall back to LooseVersion + except ValueError: + return LooseVersion(version) + + +def packages_from_names(data): + """ + Returns a list of Package objects from a string of names + """ + + from djeuscan.models import Package + + packages = [] + data = data.replace("\r", "") + + for pkg in data.split('\n'): + if '/' in pkg: + cat, pkg = pkg.split('/') + packages.extend(Package.objects.filter(category=cat, name=pkg)) + else: + packages.extend(Package.objects.filter(name=pkg)) + return packages + + +def rename_fields(vqs, fields): + ret = [] + for n in vqs: + for tr in fields: + if tr[0] in n: + n[tr[1]] = n[tr[0]] + del n[tr[0]] + ret.append(n) + return ret + + +class catch_and_return(object): + def __init__(self, err, response): + self.err = err + self.response = response + + def __call__(self, fn): + def wrapper(*args, **kwargs): + try: + return fn(*args, **kwargs) + except self.err: + return self.response + return wrapper + + +def get_account_categories(user): + from djeuscan.models import Package, CategoryAssociation + + # TODO: This is quite ugly + category_names = [obj.category for obj in + CategoryAssociation.objects.filter(user=user)] + return [c for c in Package.objects.categories() + if c["category"] in category_names] + + +def get_account_herds(user): + from djeuscan.models import Package, HerdAssociation + + ids = [obj.herd.pk for obj in + HerdAssociation.objects.filter(user=user)] + return Package.objects.herds(ids=ids) + + +def get_account_maintainers(user): + from djeuscan.models import Package, MaintainerAssociation + + ids = [obj.maintainer.pk for obj in + MaintainerAssociation.objects.filter(user=user)] + return Package.objects.maintainers(ids=ids) + + +def get_account_packages(user): + from djeuscan.models import PackageAssociation + return [obj.package for obj in + PackageAssociation.objects.filter(user=user)] From 266838b308e3bacb777c97facdfa2d9be96a7953 Mon Sep 17 00:00:00 2001 From: volpino Date: Tue, 24 Jul 2012 09:03:14 +0200 Subject: [PATCH 08/17] euscanwww: == None -> is None Signed-off-by: volpino --- euscanwww/djeuscan/processing/scan/scan_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/euscanwww/djeuscan/processing/scan/scan_metadata.py b/euscanwww/djeuscan/processing/scan/scan_metadata.py index ee028d4..54dd9c4 100644 --- a/euscanwww/djeuscan/processing/scan/scan_metadata.py +++ b/euscanwww/djeuscan/processing/scan/scan_metadata.py @@ -170,7 +170,7 @@ def scan_metadata(packages=None, category=None, logger=None, populate=False): if category: packages = Package.objects.filter(category=category) - elif packages == None: + elif packages is None: packages = Package.objects.all() if populate: From c35065e344b49fabe11a312668641c471c39fbb9 Mon Sep 17 00:00:00 2001 From: volpino Date: Tue, 24 Jul 2012 15:02:36 +0200 Subject: [PATCH 09/17] euscan: Added watch handler, fixed generic one Signed-off-by: volpino --- bin/euscan_patch_metadata | 91 +++++++++++++-------- euscanwww/djeuscan/views.py | 2 +- pym/euscan/handlers/cpan.py | 2 +- pym/euscan/handlers/generic.py | 14 ++-- pym/euscan/handlers/github.py | 2 +- pym/euscan/handlers/kde.py | 4 +- pym/euscan/handlers/php.py | 2 +- pym/euscan/handlers/pypi.py | 2 +- pym/euscan/handlers/rubygem.py | 2 +- pym/euscan/handlers/watch.py | 139 +++++++++++++++++++++++++++++++++ pym/euscan/helpers.py | 2 + 11 files changed, 213 insertions(+), 49 deletions(-) create mode 100644 pym/euscan/handlers/watch.py diff --git a/bin/euscan_patch_metadata b/bin/euscan_patch_metadata index b5649ed..0de4239 100755 --- a/bin/euscan_patch_metadata +++ b/bin/euscan_patch_metadata @@ -11,6 +11,7 @@ import logging import shutil import subprocess +from portage.exception import AmbiguousPackageName from gentoolkit.query import Query from BeautifulSoup import BeautifulSoup, SoupStrainer @@ -133,40 +134,60 @@ def get_deb_url(name): def patch_metadata(metadata_path, watch_data, diff=False): - watch_data = "\n".join([line for line in watch_data.split("\n") - if not line.startswith("#")]) # comments - watch_data = watch_data.replace("\\\n", "") # remove backslashes - watch_data = " ".join(watch_data.split()) # remove extra spaces and \n - - result = re.match( - r'(version=\d+?) (?:opts=(?:"([^"]+?)"|([^\s]+?)) )?(.*)', watch_data - ) - - version, attrs_quote, attrs, url = result.groups() - attrs = attrs_quote or attrs - - if attrs: - attrs = [x.replace('=', '="') + '"' for x in attrs.split(",")] - attrs = " ".join(attrs) + logger.info(" Patching metadata file") with open(metadata_path) as fp: original = fp.read() rindent, indent = guess_indent_values(original) - data = original - logger.info(" Patching metadata file") + # clean watch_data + watch_data = "\n".join([line for line in watch_data.split("\n") + if not line.startswith("#")]) # comments - if attrs: - watch_tag = '%s%s' % (indent, version, attrs, url) - else: - watch_tag = '%s%s' % (indent, version, url) + watch_data = watch_data.replace("\\\n", "") # remove backslashes + + watch_tags = [] + + for watch_line in watch_data.split("\n"): # there can be multiple lines + watch_line = " ".join(watch_line.split()) # remove extra spaces and \n + + version_parse = re.match("version=(\d+?)", watch_line) + if version_parse: + version = version_parse.group(1) + continue + + if not watch_line: # skip empty lines + continue + + # parse watch_line + result = re.match( + r'(?:opts=(?:"([^"]+?)"|([^\s]+?)) )?(.*)', + watch_line + ) + + attrs_quote, attrs, url = result.groups() + attrs = attrs_quote or attrs + + if attrs: + attrs = [x.replace('=', '="') + '"' for x in attrs.split(",")] + attrs = " ".join(attrs) + + if attrs: + watch_tag = '%s%s' % \ + (indent, version, attrs, url) + else: + watch_tag = '%s%s' % \ + (indent, version, url) + watch_tags.append(watch_tag) + + watch_tags = "\n".join(watch_tags) if '' in data: - data = data.replace('', '\n%s' % watch_tag, 1) + data = data.replace('', '\n%s' % watch_tags, 1) else: rep = '%s\n%s\n%s\n' % \ - (rindent, watch_tag, rindent) + (rindent, watch_tags, rindent) data = data.replace('', rep, 1) if not diff: @@ -183,14 +204,18 @@ def patch_metadata(metadata_path, watch_data, diff=False): def process_package(query, diff=False): - matches = Query(query).smart_find( - in_installed=True, - in_porttree=True, - in_overlay=True, - include_masked=True, - show_progress=False, - no_matches_fatal=False, - ) + try: + matches = Query(query).smart_find( + in_installed=True, + in_porttree=True, + in_overlay=True, + include_masked=True, + show_progress=False, + no_matches_fatal=False, + ) + except AmbiguousPackageName: + logger.error(" Ambiguous package name") + return None if len(matches) == 0: logger.error(" Package not found") @@ -224,7 +249,9 @@ def main(): for package in packages: logger.info("Processing %s..." % package) - print process_package(package, opts.diff) + result = process_package(package, opts.diff) + if result: + print result if __name__ == "__main__": main() diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py index af7d7be..3d96cb0 100644 --- a/euscanwww/djeuscan/views.py +++ b/euscanwww/djeuscan/views.py @@ -250,7 +250,7 @@ def package(request, category, package): 'upstream': upstream, 'log': log, 'vlog': vlog, - 'msg' : msg, + 'msg': msg, 'last_scan': last_scan, 'favourited': favourited, 'refreshed': refreshed, diff --git a/pym/euscan/handlers/cpan.py b/pym/euscan/handlers/cpan.py index 091c64c..a54641f 100644 --- a/pym/euscan/handlers/cpan.py +++ b/pym/euscan/handlers/cpan.py @@ -7,7 +7,7 @@ from euscan import helpers, output HANDLER_NAME = "cpan" CONFIDENCE = 100.0 -PRIORITY = 100 +PRIORITY = 90 _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*") diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/generic.py index a9a3048..831dfce 100644 --- a/pym/euscan/handlers/generic.py +++ b/pym/euscan/handlers/generic.py @@ -1,3 +1,4 @@ +from urlparse import urljoin import urllib2 import re import StringIO @@ -34,8 +35,7 @@ def scan_html(data, url, pattern): match = re.match(pattern, href, re.I) if match: - results.append((match.group(1), match.group(0))) - + results.append((".".join(match.groups()), match.group(0))) return results @@ -47,7 +47,7 @@ def scan_ftp(data, url, pattern): line = line.replace("\n", "").replace("\r", "") match = re.search(pattern, line, re.I) if match: - results.append((match.group(1), match.group(0))) + results.append((".".join(match.groups()), match.group(0))) return results @@ -77,7 +77,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): results = [] - if re.search("<\s*a\s+[^>]*href", data): + if re.search("<\s*a\s+[^>]*href", data, re.I): results.extend(scan_html(data, url, pattern)) elif url.startswith('ftp://'): results.extend(scan_ftp(data, url, pattern)) @@ -88,11 +88,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): pv = helpers.gentoo_mangle_version(up_pv) if helpers.version_filtered(cp, ver, pv): continue - - if not url.endswith('/') and not path.startswith('/'): - path = url + '/' + path - else: - path = url + path + path = urljoin(url, path) if not steps and path not in orig_url: versions.append((path, pv, HANDLER_NAME, CONFIDENCE)) diff --git a/pym/euscan/handlers/github.py b/pym/euscan/handlers/github.py index 76c50c9..9bb5596 100644 --- a/pym/euscan/handlers/github.py +++ b/pym/euscan/handlers/github.py @@ -8,7 +8,7 @@ from euscan import helpers, output HANDLER_NAME = "github" CONFIDENCE = 100.0 -PRIORITY = 100 +PRIORITY = 90 def can_handle(pkg, url): diff --git a/pym/euscan/handlers/kde.py b/pym/euscan/handlers/kde.py index 1dcead6..21722bb 100644 --- a/pym/euscan/handlers/kde.py +++ b/pym/euscan/handlers/kde.py @@ -1,6 +1,6 @@ from euscan.handlers import generic -PRIORITY = 100 +PRIORITY = 90 HANDLER_NAME = "kde" @@ -14,7 +14,7 @@ def can_handle(pkg, url): def clean_results(results): ret = [] - for path, version, confidence in results: + for path, version, _, confidence in results: if version == '5SUMS': continue ret.append((path, version, HANDLER_NAME, confidence)) diff --git a/pym/euscan/handlers/php.py b/pym/euscan/handlers/php.py index 36bf3d7..6b74ff6 100644 --- a/pym/euscan/handlers/php.py +++ b/pym/euscan/handlers/php.py @@ -7,7 +7,7 @@ from euscan import helpers, output HANDLER_NAME = "php" CONFIDENCE = 100.0 -PRIORITY = 100 +PRIORITY = 90 def can_handle(pkg, url): diff --git a/pym/euscan/handlers/pypi.py b/pym/euscan/handlers/pypi.py index 74aaeb3..9cd1620 100644 --- a/pym/euscan/handlers/pypi.py +++ b/pym/euscan/handlers/pypi.py @@ -7,7 +7,7 @@ from euscan import helpers, output HANDLER_NAME = "pypi" CONFIDENCE = 100.0 -PRIORITY = 100 +PRIORITY = 90 def can_handle(pkg, url): diff --git a/pym/euscan/handlers/rubygem.py b/pym/euscan/handlers/rubygem.py index 529e6d4..39e2334 100644 --- a/pym/euscan/handlers/rubygem.py +++ b/pym/euscan/handlers/rubygem.py @@ -7,7 +7,7 @@ from euscan import helpers, output HANDLER_NAME = "rubygem" CONFIDENCE = 100.0 -PRIORITY = 100 +PRIORITY = 90 def can_handle(pkg, url): diff --git a/pym/euscan/handlers/watch.py b/pym/euscan/handlers/watch.py new file mode 100644 index 0000000..d172072 --- /dev/null +++ b/pym/euscan/handlers/watch.py @@ -0,0 +1,139 @@ +import re +import urllib2 + +import portage + +from euscan.handlers import generic +from euscan import output, helpers + +PRIORITY = 100 + +HANDLER_NAME = "watch" +CONFIDENCE = 100.0 + + +is_pattern = r"\([^\/]+\)" + + +def can_handle(pkg, url): + try: + return pkg.metadata._xml_tree.find("upstream").find("watch") \ + is not None + except AttributeError: + return False + + +def parse_mangles(mangles, string): + for mangle in mangles: + # convert regex from perl format to python format + m = re.match(r"s/(.*[^\\])/(.*)/", mangle) + pattern, repl = m.groups() + repl = re.sub(r"\$(\d+)", r"\\\1", repl) + string = re.sub(pattern, repl, string) + return string + + +def clean_results(results, versionmangle, urlmangle): + ret = [] + + for path, version, _, _ in results: + version = parse_mangles(versionmangle, version) + path = parse_mangles(urlmangle, path) + ret.append((path, version, HANDLER_NAME, CONFIDENCE)) + + return ret + + +def parse_watch(pkg): + for watch_tag in pkg.metadata._xml_tree.find("upstream").findall("watch"): + try: + base, file_pattern = watch_tag.text.split(" ")[:2] + except ValueError: + base, file_pattern = watch_tag.text, None + + # the file pattern can be in the base url + pattern_regex = r"/([^/]*\([^/]*\)[^/]*)$" + match = re.search(pattern_regex, base) + if match: + file_pattern = match.group(1) + base = base.replace(file_pattern, "") + + # handle sf.net specially + base = base.replace( + "http://sf.net/", "http://qa.debian.org/watch/sf.php/" + ) + + vmangle = watch_tag.attrib.get("uversionmangle", None) or \ + watch_tag.attrib.get("versionmangle", None) + versionmangle = vmangle.split(";") if vmangle else [] + + umangle = watch_tag.attrib.get("downloadurlmangle", None) + urlmangle = umangle.split(";") if umangle else [] + + yield (base, file_pattern, versionmangle, urlmangle) + + +def handle_directory_patterns(base, file_pattern): + """ + Directory pattern matching + e.g.: base: ftp://ftp.nessus.org/pub/nessus/nessus-([\d\.]+)/src/ + file_pattern: nessus-core-([\d\.]+)\.tar\.gz + """ + splitted = base.split("/") + i = 0 + basedir = [] + for elem in splitted: + if re.search(is_pattern, elem): + break + basedir.append(elem) + i += 1 + basedir = "/".join(basedir) + directory_pattern = splitted[i] + final = "/".join(splitted[i + 1:]) + + try: + fp = helpers.urlopen(basedir) + except urllib2.URLError: + return [] + except IOError: + return [] + + if not fp: + return [] + + data = fp.read() + + if basedir.startswith("ftp://"): + scan_data = generic.scan_ftp(data, basedir, directory_pattern) + else: + scan_data = generic.scan_html(data, basedir, directory_pattern) + + return [("/".join((basedir, path, final)), file_pattern) + for _, path in scan_data] + + +def scan(pkg, url): + output.einfo("Using watch data") + + cp, ver, rev = portage.pkgsplit(pkg.cpv) + + results = [] + for base, file_pattern, versionmangle, urlmangle in parse_watch(pkg): + if not re.search(is_pattern, base): + steps = [(base, file_pattern)] + res = generic.scan_directory_recursive( + cp, ver, rev, "", steps, url + ) + else: + res = [] + for step in handle_directory_patterns(base, file_pattern): + res += generic.scan_directory_recursive( + cp, ver, rev, "", [step], url + ) + + results += clean_results(res, versionmangle, urlmangle) + return results + + +def brute_force(pkg, url): + return [] diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py index 1e385cd..6582393 100644 --- a/pym/euscan/helpers.py +++ b/pym/euscan/helpers.py @@ -33,6 +33,7 @@ VERSION_CMP_PACKAGE_QUIRKS = { _v_end = '((-|_)(pre|p|beta|b|alpha|a|rc|r)\d*)' _v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))' + # Stolen from g-pypi def gentoo_mangle_version(up_pv): """Convert PV to MY_PV if needed @@ -537,6 +538,7 @@ def generate_scan_paths(url): return steps + def parse_mirror(uri): from random import shuffle From 29101f88f8f251082380830aac7bb398c42b16f2 Mon Sep 17 00:00:00 2001 From: volpino Date: Wed, 25 Jul 2012 10:19:27 +0200 Subject: [PATCH 10/17] euscanwww: Removing useless stuff while creating watch tag * Removing useless options or actions * Fixing handling of substitution regex Signed-off-by: volpino --- bin/euscan_patch_metadata | 24 +++++++++++++++++------- pym/euscan/handlers/watch.py | 4 ++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/bin/euscan_patch_metadata b/bin/euscan_patch_metadata index 0de4239..642b639 100755 --- a/bin/euscan_patch_metadata +++ b/bin/euscan_patch_metadata @@ -166,16 +166,26 @@ def patch_metadata(metadata_path, watch_data, diff=False): watch_line ) - attrs_quote, attrs, url = result.groups() - attrs = attrs_quote or attrs + opts_quote, opts, url = result.groups() + opts = opts_quote or opts - if attrs: - attrs = [x.replace('=', '="') + '"' for x in attrs.split(",")] - attrs = " ".join(attrs) + if opts: + # clean opts, skip useless ones + valid = ("uversionmangle", "versionmangle", "downloadurlmangle") + cleaned_opts = [] + for opt in opts.split(","): + opt_name, opt_value = opt.split("=") + if opt_name in valid: + cleaned_opts.append('%s="%s"' % (opt_name, opt_value)) + opts = " ".join(cleaned_opts) - if attrs: + # clean url from useless stuff. Just keep [] + url_search = re.search(r"^([^\s]+)(?: ([^\s]*\([^\s]+\)[^\s]*))?", url) + url = " ".join([x for x in url_search.groups() if x is not None]) + + if opts: watch_tag = '%s%s' % \ - (indent, version, attrs, url) + (indent, version, opts, url) else: watch_tag = '%s%s' % \ (indent, version, url) diff --git a/pym/euscan/handlers/watch.py b/pym/euscan/handlers/watch.py index d172072..a129281 100644 --- a/pym/euscan/handlers/watch.py +++ b/pym/euscan/handlers/watch.py @@ -26,7 +26,11 @@ def can_handle(pkg, url): def parse_mangles(mangles, string): for mangle in mangles: # convert regex from perl format to python format + # there are some regex in this format: s/pattern/replacement/ m = re.match(r"s/(.*[^\\])/(.*)/", mangle) + if not m: + # or in this format s|pattern|replacement| + m = re.match(r"s\|(.*[^\\])\|(.*)\|", mangle) pattern, repl = m.groups() repl = re.sub(r"\$(\d+)", r"\\\1", repl) string = re.sub(pattern, repl, string) From 21c6a9107d08f8f02d3960a4be30838c5a04525d Mon Sep 17 00:00:00 2001 From: volpino Date: Wed, 25 Jul 2012 10:44:15 +0200 Subject: [PATCH 11/17] euscan: generic handler fixes * urljoin was broken with dirs ("http://site.com/lol" joined with "wat" produces http://site.com/wat) * fixed _v regex, don't match what is not needed Signed-off-by: volpino --- pym/euscan/handlers/generic.py | 13 ++++++++++--- pym/euscan/helpers.py | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/generic.py index 831dfce..12c4a99 100644 --- a/pym/euscan/handlers/generic.py +++ b/pym/euscan/handlers/generic.py @@ -35,7 +35,10 @@ def scan_html(data, url, pattern): match = re.match(pattern, href, re.I) if match: - results.append((".".join(match.groups()), match.group(0))) + results.append( + (".".join([x for x in match.groups() if x is not None]), + match.group(0)) + ) return results @@ -47,8 +50,10 @@ def scan_ftp(data, url, pattern): line = line.replace("\n", "").replace("\r", "") match = re.search(pattern, line, re.I) if match: - results.append((".".join(match.groups()), match.group(0))) - + results.append( + (".".join([x for x in match.groups() if x is not None]), + match.group(0)) + ) return results @@ -88,6 +93,8 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): pv = helpers.gentoo_mangle_version(up_pv) if helpers.version_filtered(cp, ver, pv): continue + if not url.endswith("/"): + url = url + "/" path = urljoin(url, path) if not steps and path not in orig_url: diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py index 6582393..ec721b7 100644 --- a/pym/euscan/helpers.py +++ b/pym/euscan/helpers.py @@ -30,8 +30,8 @@ VERSION_CMP_PACKAGE_QUIRKS = { 'sys-process/htop': htop_vercmp } -_v_end = '((-|_)(pre|p|beta|b|alpha|a|rc|r)\d*)' -_v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))' +_v_end = r'(?:(?:-|_)(?:pre|p|beta|b|alpha|a|rc|r)\d*)' +_v = r'((?:\d+)(?:(?:\.\d+)*)(?:[a-zA-Z]*?)(?:' + _v_end + '*))' # Stolen from g-pypi From 36e1aa6d12b0f2816e0446c03d2de7dc92750d1c Mon Sep 17 00:00:00 2001 From: volpino Date: Wed, 25 Jul 2012 13:49:47 +0200 Subject: [PATCH 12/17] euscan: euscan_path_metadata diff using difflib Signed-off-by: volpino --- bin/euscan_patch_metadata | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/bin/euscan_patch_metadata b/bin/euscan_patch_metadata index 642b639..617ca3b 100755 --- a/bin/euscan_patch_metadata +++ b/bin/euscan_patch_metadata @@ -9,7 +9,7 @@ import tarfile import gzip import logging import shutil -import subprocess +from difflib import unified_diff from portage.exception import AmbiguousPackageName from gentoolkit.query import Query @@ -133,9 +133,11 @@ def get_deb_url(name): return deb_url, deb_type -def patch_metadata(metadata_path, watch_data, diff=False): +def patch_metadata(package, watch_data, diff=False): logger.info(" Patching metadata file") + metadata_path = package.metadata.metadata_path + with open(metadata_path) as fp: original = fp.read() rindent, indent = guess_indent_values(original) @@ -203,14 +205,17 @@ def patch_metadata(metadata_path, watch_data, diff=False): if not diff: return data else: - _, data_path = mkstemp() - with open(data_path, "w") as f: - f.write(data) - sub = subprocess.Popen(["diff", metadata_path, data_path], - stdout=subprocess.PIPE) - content = sub.stdout.read() - os.unlink(data_path) - return content + # Generate clean a/category/package/metadata.xml path + n = metadata_path.find(package.category) + if n != -1: + metadata_path = metadata_path[n:] + res = unified_diff( + original.splitlines(True), + data.splitlines(True), + fromfile=os.path.join('a/', metadata_path), + tofile=os.path.join('b/', metadata_path), + ) + return "".join([x for x in res]) def process_package(query, diff=False): @@ -236,12 +241,11 @@ def process_package(query, diff=False): if '9999' in package.version and len(matches) > 0: package = matches.pop() - metadata_path = package.metadata.metadata_path watch_data = get_watch_data(package) if watch_data is None: logger.error(" No watch file found") else: - return patch_metadata(metadata_path, watch_data, diff=diff) + return patch_metadata(package, watch_data, diff=diff) def main(): @@ -261,7 +265,7 @@ def main(): logger.info("Processing %s..." % package) result = process_package(package, opts.diff) if result: - print result + sys.stdout.write(result) if __name__ == "__main__": main() From 12bf1fc75b44857f83f760ce613567ec7486efbd Mon Sep 17 00:00:00 2001 From: volpino Date: Thu, 26 Jul 2012 10:44:10 +0200 Subject: [PATCH 13/17] euscan: Redesigning the handlers layout Signed-off-by: volpino --- bin/euscan | 4 +- pym/euscan/handlers/__init__.py | 103 +++++++++++++++------ pym/euscan/handlers/package/__init__.py | 19 ++++ pym/euscan/handlers/{ => package}/watch.py | 14 +-- pym/euscan/handlers/url/__init__.py | 19 ++++ pym/euscan/handlers/{ => url}/cpan.py | 4 - pym/euscan/handlers/{ => url}/generic.py | 55 ++++++----- pym/euscan/handlers/{ => url}/github.py | 4 - pym/euscan/handlers/{ => url}/kde.py | 15 ++- pym/euscan/handlers/{ => url}/php.py | 4 - pym/euscan/handlers/{ => url}/pypi.py | 4 - pym/euscan/handlers/{ => url}/rubygem.py | 4 - pym/euscan/scan.py | 72 ++------------ 13 files changed, 168 insertions(+), 153 deletions(-) create mode 100644 pym/euscan/handlers/package/__init__.py rename pym/euscan/handlers/{ => package}/watch.py (94%) create mode 100644 pym/euscan/handlers/url/__init__.py rename pym/euscan/handlers/{ => url}/cpan.py (98%) rename pym/euscan/handlers/{ => url}/generic.py (80%) rename pym/euscan/handlers/{ => url}/github.py (97%) rename pym/euscan/handlers/{ => url}/kde.py (65%) rename pym/euscan/handlers/{ => url}/php.py (97%) rename pym/euscan/handlers/{ => url}/pypi.py (96%) rename pym/euscan/handlers/{ => url}/rubygem.py (97%) diff --git a/bin/euscan b/bin/euscan index 4469f4e..e0371b4 100755 --- a/bin/euscan +++ b/bin/euscan @@ -130,9 +130,9 @@ def print_usage(_error=None, help=None): " - Ignore non-stable versions only if current\n" + " " * 38 + "version is stable", file=out) print(yellow(" --mirror") + - " - use mirror:// URIs", file=out) + " - use mirror:// URIs", file=out) print(yellow(" --ebuild-uri") + - " - use ebuild variables in URIs", file=out) + " - use ebuild variables in URIs", file=out) print(file=out) if _error in ('packages',) or help: diff --git a/pym/euscan/handlers/__init__.py b/pym/euscan/handlers/__init__.py index 495ef7c..899ef24 100644 --- a/pym/euscan/handlers/__init__.py +++ b/pym/euscan/handlers/__init__.py @@ -1,35 +1,86 @@ -import pkgutil - -# autoimport all modules in this directory and append them to handlers list -handlers = [] -for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): - module = loader.find_module(module_name).load_module(module_name) - handlers.append(module) - -# sort handlers by priority (e.g.: generic should be run lastly) -handlers = sorted( - handlers, - key=lambda handler: handler.PRIORITY, - reverse=True -) +import sys +from euscan import CONFIG, output +from euscan.handlers.package import handlers as pkg_handlers +from euscan.handlers.url import handlers as url_handlers -def find_best_handler(pkg, url): - for handler in handlers: +def find_best_pkg_handler(pkg): + """ + Find the best handler for the given package + """ + for handler in pkg_handlers: + if handler.can_handle(pkg): + return handler + return None + + +def find_best_url_handler(pkg, url): + """ + Find the best handler for the given url + """ + for handler in url_handlers: if handler.can_handle(pkg, url): return handler return None -def scan(pkg, url): - handler = find_best_handler(pkg, url) - if handler: - return handler.scan(pkg, url) - return [] +def scan(pkg, urls, on_progress=None): + """ + Scans upstream for the given package. + First tries if a package wide handler is available, then fallbacks + in url handling. + """ + pkg_handler = find_best_pkg_handler(pkg) + if pkg_handler: + if on_progress: + on_progress(increment=35) + if not CONFIG['quiet'] and not CONFIG['format']: + sys.stdout.write("\n") -def brute_force(pkg, url): - handler = find_best_handler(pkg, url) - if handler: - return handler.brute_force(pkg, url) - return [] + versions = pkg_handler.scan(pkg) + + if on_progress: + on_progress(increment=35) + return versions + + if on_progress: + progress_available = 70 + num_urls = sum([len(urls[fn]) for fn in urls]) + if num_urls > 0: + progress_increment = progress_available / num_urls + else: + progress_increment = 0 + + versions = [] + + for filename in urls: + for url in urls[filename]: + if on_progress and progress_available > 0: + on_progress(increment=progress_increment) + progress_available -= progress_increment + + if not CONFIG['quiet'] and not CONFIG['format']: + sys.stdout.write("\n") + output.einfo("SRC_URI is '%s'" % url) + + if '://' not in url: + output.einfo("Invalid url '%s'" % url) + continue + + try: + url_handler = find_best_url_handler(pkg, url) + versions.extend(url_handler.scan(pkg, url)) + except Exception as e: + output.ewarn( + "Handler failed: [%s] %s" % + (e.__class__.__name__, e.message) + ) + + if versions and CONFIG['oneshot']: + break + + if on_progress and progress_available > 0: + on_progress(increment=progress_available) + + return versions diff --git a/pym/euscan/handlers/package/__init__.py b/pym/euscan/handlers/package/__init__.py new file mode 100644 index 0000000..8530b10 --- /dev/null +++ b/pym/euscan/handlers/package/__init__.py @@ -0,0 +1,19 @@ +""" +Package wide handlers for scanning upstream +""" + +import pkgutil + +handlers = [] + +# autoimport all modules in this directory and append them to handlers list +for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): + module = loader.find_module(module_name).load_module(module_name) + handlers.append(module) + +# sort handlers by priority +handlers = sorted( + handlers, + key=lambda handler: handler.PRIORITY, + reverse=True +) diff --git a/pym/euscan/handlers/watch.py b/pym/euscan/handlers/package/watch.py similarity index 94% rename from pym/euscan/handlers/watch.py rename to pym/euscan/handlers/package/watch.py index a129281..14f25d2 100644 --- a/pym/euscan/handlers/watch.py +++ b/pym/euscan/handlers/package/watch.py @@ -3,7 +3,7 @@ import urllib2 import portage -from euscan.handlers import generic +from euscan.handlers.url import generic from euscan import output, helpers PRIORITY = 100 @@ -15,7 +15,7 @@ CONFIDENCE = 100.0 is_pattern = r"\([^\/]+\)" -def can_handle(pkg, url): +def can_handle(pkg): try: return pkg.metadata._xml_tree.find("upstream").find("watch") \ is not None @@ -116,7 +116,7 @@ def handle_directory_patterns(base, file_pattern): for _, path in scan_data] -def scan(pkg, url): +def scan(pkg): output.einfo("Using watch data") cp, ver, rev = portage.pkgsplit(pkg.cpv) @@ -126,18 +126,14 @@ def scan(pkg, url): if not re.search(is_pattern, base): steps = [(base, file_pattern)] res = generic.scan_directory_recursive( - cp, ver, rev, "", steps, url + cp, ver, rev, "", steps, base ) else: res = [] for step in handle_directory_patterns(base, file_pattern): res += generic.scan_directory_recursive( - cp, ver, rev, "", [step], url + cp, ver, rev, "", [step], base ) results += clean_results(res, versionmangle, urlmangle) return results - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/handlers/url/__init__.py b/pym/euscan/handlers/url/__init__.py new file mode 100644 index 0000000..7328644 --- /dev/null +++ b/pym/euscan/handlers/url/__init__.py @@ -0,0 +1,19 @@ +""" +Url wide handlers for scanning upstream +""" + +import pkgutil + +handlers = [] + +# autoimport all modules in this directory and append them to handlers list +for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): + module = loader.find_module(module_name).load_module(module_name) + handlers.append(module) + +# sort handlers by priority +handlers = sorted( + handlers, + key=lambda handler: handler.PRIORITY, + reverse=True +) diff --git a/pym/euscan/handlers/cpan.py b/pym/euscan/handlers/url/cpan.py similarity index 98% rename from pym/euscan/handlers/cpan.py rename to pym/euscan/handlers/url/cpan.py index a54641f..0f587e1 100644 --- a/pym/euscan/handlers/cpan.py +++ b/pym/euscan/handlers/url/cpan.py @@ -131,7 +131,3 @@ def scan(pkg, url): ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/url/generic.py similarity index 80% rename from pym/euscan/handlers/generic.py rename to pym/euscan/handlers/url/generic.py index 12c4a99..38cfa10 100644 --- a/pym/euscan/handlers/generic.py +++ b/pym/euscan/handlers/url/generic.py @@ -108,43 +108,50 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): def scan(pkg, url): - for bu in SCANDIR_BLACKLIST_URLS: - if re.match(bu, url): - output.einfo("%s is blacklisted by rule %s" % (url, bu)) + if CONFIG["scan-dir"]: + for bu in SCANDIR_BLACKLIST_URLS: + if re.match(bu, url): + output.einfo("%s is blacklisted by rule %s" % (url, bu)) + return [] + + resolved_url = helpers.parse_mirror(url) + if not resolved_url: return [] - resolved_url = helpers.parse_mirror(url) - if not resolved_url: - return [] + cp, ver, rev = portage.pkgsplit(pkg.cpv) - cp, ver, rev = portage.pkgsplit(pkg.cpv) + # 'Hack' for _beta/_rc versions where _ is used instead of - + if ver not in resolved_url: + newver = helpers.version_change_end_sep(ver) + if newver and newver in resolved_url: + output.einfo( + "Version: using %s instead of %s" % (newver, ver) + ) + ver = newver - # 'Hack' for _beta/_rc versions where _ is used instead of - - if ver not in resolved_url: - newver = helpers.version_change_end_sep(ver) - if newver and newver in resolved_url: + template = helpers.template_from_url(resolved_url, ver) + if '${' not in template: output.einfo( - "Version: using %s instead of %s" % (newver, ver) + "Url doesn't seems to depend on version: %s not found in %s" % + (ver, resolved_url) ) - ver = newver + return [] + else: + output.einfo("Scanning: %s" % template) - template = helpers.template_from_url(resolved_url, ver) - if '${' not in template: - output.einfo( - "Url doesn't seems to depend on version: %s not found in %s" % - (ver, resolved_url) - ) - return [] - else: - output.einfo("Scanning: %s" % template) + steps = helpers.generate_scan_paths(template) + ret = scan_directory_recursive(cp, ver, rev, "", steps, url) - steps = helpers.generate_scan_paths(template) - ret = scan_directory_recursive(cp, ver, rev, "", steps, url) + if not ret: + brute_force(pkg, url) return ret def brute_force(pkg, url): + if CONFIG["brute-force"] == 0: + return [] + cp, ver, rev = portage.pkgsplit(pkg.cpv) url = helpers.parse_mirror(url) diff --git a/pym/euscan/handlers/github.py b/pym/euscan/handlers/url/github.py similarity index 97% rename from pym/euscan/handlers/github.py rename to pym/euscan/handlers/url/github.py index 9bb5596..080a559 100644 --- a/pym/euscan/handlers/github.py +++ b/pym/euscan/handlers/url/github.py @@ -52,7 +52,3 @@ def scan(pkg, url): if helpers.version_filtered(cp, ver, pv): continue yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE) - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/handlers/kde.py b/pym/euscan/handlers/url/kde.py similarity index 65% rename from pym/euscan/handlers/kde.py rename to pym/euscan/handlers/url/kde.py index 21722bb..5535158 100644 --- a/pym/euscan/handlers/kde.py +++ b/pym/euscan/handlers/url/kde.py @@ -1,4 +1,4 @@ -from euscan.handlers import generic +from euscan.handlers.url import generic PRIORITY = 90 @@ -29,14 +29,11 @@ def scan(pkg, url): url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/') results += generic.scan(pkg.cpv, url) - return clean_results(results) + if not results: # if nothing was found go brute forcing + results = generic.brute_force(pkg.cpv, url) - -def brute_force(pkg, url): - results = generic.brute_force(pkg.cpv, url) - - if url.startswith('mirror://kde/unstable/'): - url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/') - results += generic.brute_force(pkg.cpv, url) + if url.startswith('mirror://kde/unstable/'): + url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/') + results += generic.brute_force(pkg.cpv, url) return clean_results(results) diff --git a/pym/euscan/handlers/php.py b/pym/euscan/handlers/url/php.py similarity index 97% rename from pym/euscan/handlers/php.py rename to pym/euscan/handlers/url/php.py index 6b74ff6..853059a 100644 --- a/pym/euscan/handlers/php.py +++ b/pym/euscan/handlers/url/php.py @@ -70,7 +70,3 @@ def scan(pkg, url): ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/handlers/pypi.py b/pym/euscan/handlers/url/pypi.py similarity index 96% rename from pym/euscan/handlers/pypi.py rename to pym/euscan/handlers/url/pypi.py index 9cd1620..8ed8021 100644 --- a/pym/euscan/handlers/pypi.py +++ b/pym/euscan/handlers/url/pypi.py @@ -52,7 +52,3 @@ def scan(pkg, url): ret.append((urls, pv, HANDLER_NAME, CONFIDENCE)) return ret - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/handlers/rubygem.py b/pym/euscan/handlers/url/rubygem.py similarity index 97% rename from pym/euscan/handlers/rubygem.py rename to pym/euscan/handlers/url/rubygem.py index 39e2334..950e81b 100644 --- a/pym/euscan/handlers/rubygem.py +++ b/pym/euscan/handlers/url/rubygem.py @@ -71,7 +71,3 @@ def scan(pkg, url): ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret - - -def brute_force(pkg, url): - return [] diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py index 114c81e..5d9fd06 100644 --- a/pym/euscan/scan.py +++ b/pym/euscan/scan.py @@ -44,60 +44,6 @@ def filter_versions(cp, versions): ] -def scan_upstream_urls(pkg, urls, on_progress): - versions = [] - - if on_progress: - progress_available = 70 - num_urls = sum([len(urls[fn]) for fn in urls]) - if num_urls > 0: - progress_increment = progress_available / num_urls - else: - progress_increment = 0 - - for filename in urls: - for url in urls[filename]: - - if on_progress and progress_available > 0: - on_progress(increment=progress_increment) - progress_available -= progress_increment - - if not CONFIG['quiet'] and not CONFIG['format']: - pp.uprint() - output.einfo("SRC_URI is '%s'" % url) - - if '://' not in url: - output.einfo("Invalid url '%s'" % url) - continue - - # Try normal scan - if CONFIG["scan-dir"]: - try: - versions.extend(handlers.scan(pkg, url)) - except Exception as e: - output.ewarn("Handler failed: [%s] %s" - % (e.__class__.__name__, e.message)) - - if versions and CONFIG['oneshot']: - break - - # Brute Force - if CONFIG["brute-force"] > 0: - versions.extend(handlers.brute_force(pkg, url)) - - if versions and CONFIG['oneshot']: - break - - cp, ver, rev = portage.pkgsplit(pkg.cpv) - - result = filter_versions(cp, versions) - - if on_progress and progress_available > 0: - on_progress(increment=progress_available) - - return result - - # gentoolkit stores PORTDB, so even if we modify it to add an overlay # it will still use the old dbapi def reload_gentoolkit(): @@ -120,7 +66,6 @@ def scan_upstream(query, on_progress=None): """ Scans the upstream searching new versions for the given query """ - matches = [] if query.endswith(".ebuild"): @@ -188,10 +133,6 @@ def scan_upstream(query, on_progress=None): output.metadata("description", pkg.environment("DESCRIPTION")) cpv = pkg.cpv - - _, _, ver, _ = portage.catpkgsplit(cpv) - is_current_version_stable = is_version_stable(ver) - metadata = { "EAPI": portage.settings["EAPI"], "SRC_URI": pkg.environment("SRC_URI", False), @@ -212,15 +153,20 @@ def scan_upstream(query, on_progress=None): else: urls = alist - # output scan time for formatted output - scan_time = (datetime.now() - start_time).total_seconds() - output.metadata("scan_time", scan_time, show=False) + versions = handlers.scan(pkg, urls, on_progress) - result = scan_upstream_urls(pkg, urls, on_progress) + cp, ver, rev = portage.pkgsplit(pkg.cpv) + + result = filter_versions(cp, versions) if on_progress: on_progress(increment=10) + # output scan time for formatted output + scan_time = (datetime.now() - start_time).total_seconds() + output.metadata("scan_time", scan_time, show=False) + + is_current_version_stable = is_version_stable(ver) if len(result) > 0: if not (CONFIG['format'] or CONFIG['quiet']): print("\n", file=sys.stderr) From 7b9560ced01c53723631917a49387570aa8a24eb Mon Sep 17 00:00:00 2001 From: volpino Date: Thu, 26 Jul 2012 14:37:03 +0200 Subject: [PATCH 14/17] euscanwww: Moving from XXXAssociation to UserProfile Signed-off-by: volpino --- euscanwww/djeuscan/admin.py | 9 +- euscanwww/djeuscan/helpers.py | 31 +- ...__del_unique_maintainerassociation_user.py | 355 ++++++++++++++++++ ..._overlay_name__add_unique_category_name.py | 197 ++++++++++ euscanwww/djeuscan/models.py | 80 ++-- .../processing/misc/update_counters.py | 23 +- euscanwww/djeuscan/tests/euscan_factory.py | 39 +- euscanwww/djeuscan/tests/test_views.py | 42 ++- euscanwww/djeuscan/views.py | 122 ++---- euscanwww/euscanwww/settings.py | 2 + euscanwww/runtests.py | 1 + 11 files changed, 715 insertions(+), 186 deletions(-) create mode 100644 euscanwww/djeuscan/migrations/0016_auto__del_maintainerassociation__del_unique_maintainerassociation_user.py create mode 100644 euscanwww/djeuscan/migrations/0017_auto__add_unique_overlay_name__add_unique_category_name.py diff --git a/euscanwww/djeuscan/admin.py b/euscanwww/djeuscan/admin.py index 0322e18..a24870e 100644 --- a/euscanwww/djeuscan/admin.py +++ b/euscanwww/djeuscan/admin.py @@ -1,7 +1,6 @@ from djeuscan.models import Package, Version, VersionLog, EuscanResult, \ Log, WorldLog, CategoryLog, HerdLog, MaintainerLog, Herd, Maintainer, \ - RefreshPackageQuery, HerdAssociation, CategoryAssociation, \ - MaintainerAssociation, PackageAssociation, ProblemReport + RefreshPackageQuery, Category, Overlay, ProblemReport from django.contrib import admin @@ -54,9 +53,7 @@ admin.site.register(HerdLog) admin.site.register(MaintainerLog) admin.site.register(RefreshPackageQuery) -admin.site.register(HerdAssociation) -admin.site.register(CategoryAssociation) -admin.site.register(MaintainerAssociation) -admin.site.register(PackageAssociation) +admin.site.register(Category) +admin.site.register(Overlay) admin.site.register(ProblemReport, ProblemReportAdmin) diff --git a/euscanwww/djeuscan/helpers.py b/euscanwww/djeuscan/helpers.py index f7a6b82..2a5f878 100644 --- a/euscanwww/djeuscan/helpers.py +++ b/euscanwww/djeuscan/helpers.py @@ -76,33 +76,32 @@ class catch_and_return(object): return wrapper -def get_account_categories(user): - from djeuscan.models import Package, CategoryAssociation +def get_profile(user): + from djeuscan.models import UserProfile + try: + return user.get_profile() + except UserProfile.DoesNotExist: + UserProfile.objects.create(user=user) + return user.get_profile() + +def get_account_categories(user): + from djeuscan.models import Package # TODO: This is quite ugly - category_names = [obj.category for obj in - CategoryAssociation.objects.filter(user=user)] + category_names = [obj.name for obj in get_profile(user).categories.all()] return [c for c in Package.objects.categories() if c["category"] in category_names] def get_account_herds(user): - from djeuscan.models import Package, HerdAssociation + from djeuscan.models import Package - ids = [obj.herd.pk for obj in - HerdAssociation.objects.filter(user=user)] + ids = [herd.pk for herd in get_profile(user).herds.all()] return Package.objects.herds(ids=ids) def get_account_maintainers(user): - from djeuscan.models import Package, MaintainerAssociation + from djeuscan.models import Package - ids = [obj.maintainer.pk for obj in - MaintainerAssociation.objects.filter(user=user)] + ids = [obj.pk for obj in get_profile(user).maintainers.all()] return Package.objects.maintainers(ids=ids) - - -def get_account_packages(user): - from djeuscan.models import PackageAssociation - return [obj.package for obj in - PackageAssociation.objects.filter(user=user)] diff --git a/euscanwww/djeuscan/migrations/0016_auto__del_maintainerassociation__del_unique_maintainerassociation_user.py b/euscanwww/djeuscan/migrations/0016_auto__del_maintainerassociation__del_unique_maintainerassociation_user.py new file mode 100644 index 0000000..b4fe300 --- /dev/null +++ b/euscanwww/djeuscan/migrations/0016_auto__del_maintainerassociation__del_unique_maintainerassociation_user.py @@ -0,0 +1,355 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Removing unique constraint on 'CategoryAssociation', fields ['user', 'category'] + db.delete_unique('djeuscan_categoryassociation', ['user_id', 'category']) + + # Removing unique constraint on 'OverlayAssociation', fields ['user', 'overlay'] + db.delete_unique('djeuscan_overlayassociation', ['user_id', 'overlay']) + + # Removing unique constraint on 'HerdAssociation', fields ['user', 'herd'] + db.delete_unique('djeuscan_herdassociation', ['user_id', 'herd_id']) + + # Removing unique constraint on 'PackageAssociation', fields ['user', 'package'] + db.delete_unique('djeuscan_packageassociation', ['user_id', 'package_id']) + + # Removing unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer'] + db.delete_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id']) + + # Deleting model 'MaintainerAssociation' + db.delete_table('djeuscan_maintainerassociation') + + # Deleting model 'PackageAssociation' + db.delete_table('djeuscan_packageassociation') + + # Deleting model 'HerdAssociation' + db.delete_table('djeuscan_herdassociation') + + # Deleting model 'OverlayAssociation' + db.delete_table('djeuscan_overlayassociation') + + # Deleting model 'CategoryAssociation' + db.delete_table('djeuscan_categoryassociation') + + # Adding model 'UserProfile' + db.create_table('djeuscan_userprofile', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('user', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['auth.User'], unique=True)), + )) + db.send_create_signal('djeuscan', ['UserProfile']) + + # Adding M2M table for field herds on 'UserProfile' + db.create_table('djeuscan_userprofile_herds', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)), + ('herd', models.ForeignKey(orm['djeuscan.herd'], null=False)) + )) + db.create_unique('djeuscan_userprofile_herds', ['userprofile_id', 'herd_id']) + + # Adding M2M table for field maintainers on 'UserProfile' + db.create_table('djeuscan_userprofile_maintainers', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)), + ('maintainer', models.ForeignKey(orm['djeuscan.maintainer'], null=False)) + )) + db.create_unique('djeuscan_userprofile_maintainers', ['userprofile_id', 'maintainer_id']) + + # Adding M2M table for field packages on 'UserProfile' + db.create_table('djeuscan_userprofile_packages', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)), + ('package', models.ForeignKey(orm['djeuscan.package'], null=False)) + )) + db.create_unique('djeuscan_userprofile_packages', ['userprofile_id', 'package_id']) + + # Adding M2M table for field categories on 'UserProfile' + db.create_table('djeuscan_userprofile_categories', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)), + ('category', models.ForeignKey(orm['djeuscan.category'], null=False)) + )) + db.create_unique('djeuscan_userprofile_categories', ['userprofile_id', 'category_id']) + + # Adding M2M table for field overlays on 'UserProfile' + db.create_table('djeuscan_userprofile_overlays', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('userprofile', models.ForeignKey(orm['djeuscan.userprofile'], null=False)), + ('overlay', models.ForeignKey(orm['djeuscan.overlay'], null=False)) + )) + db.create_unique('djeuscan_userprofile_overlays', ['userprofile_id', 'overlay_id']) + + # Adding model 'Category' + db.create_table('djeuscan_category', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128)), + )) + db.send_create_signal('djeuscan', ['Category']) + + # Adding model 'Overlay' + db.create_table('djeuscan_overlay', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=128)), + )) + db.send_create_signal('djeuscan', ['Overlay']) + + def backwards(self, orm): + # Adding model 'MaintainerAssociation' + db.create_table('djeuscan_maintainerassociation', ( + ('maintainer', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Maintainer'])), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + )) + db.send_create_signal('djeuscan', ['MaintainerAssociation']) + + # Adding unique constraint on 'MaintainerAssociation', fields ['user', 'maintainer'] + db.create_unique('djeuscan_maintainerassociation', ['user_id', 'maintainer_id']) + + # Adding model 'PackageAssociation' + db.create_table('djeuscan_packageassociation', ( + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('package', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Package'])), + )) + db.send_create_signal('djeuscan', ['PackageAssociation']) + + # Adding unique constraint on 'PackageAssociation', fields ['user', 'package'] + db.create_unique('djeuscan_packageassociation', ['user_id', 'package_id']) + + # Adding model 'HerdAssociation' + db.create_table('djeuscan_herdassociation', ( + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('herd', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djeuscan.Herd'])), + )) + db.send_create_signal('djeuscan', ['HerdAssociation']) + + # Adding unique constraint on 'HerdAssociation', fields ['user', 'herd'] + db.create_unique('djeuscan_herdassociation', ['user_id', 'herd_id']) + + # Adding model 'OverlayAssociation' + db.create_table('djeuscan_overlayassociation', ( + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('overlay', self.gf('django.db.models.fields.CharField')(max_length=128)), + )) + db.send_create_signal('djeuscan', ['OverlayAssociation']) + + # Adding unique constraint on 'OverlayAssociation', fields ['user', 'overlay'] + db.create_unique('djeuscan_overlayassociation', ['user_id', 'overlay']) + + # Adding model 'CategoryAssociation' + db.create_table('djeuscan_categoryassociation', ( + ('category', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + )) + db.send_create_signal('djeuscan', ['CategoryAssociation']) + + # Adding unique constraint on 'CategoryAssociation', fields ['user', 'category'] + db.create_unique('djeuscan_categoryassociation', ['user_id', 'category']) + + # Deleting model 'UserProfile' + db.delete_table('djeuscan_userprofile') + + # Removing M2M table for field herds on 'UserProfile' + db.delete_table('djeuscan_userprofile_herds') + + # Removing M2M table for field maintainers on 'UserProfile' + db.delete_table('djeuscan_userprofile_maintainers') + + # Removing M2M table for field packages on 'UserProfile' + db.delete_table('djeuscan_userprofile_packages') + + # Removing M2M table for field categories on 'UserProfile' + db.delete_table('djeuscan_userprofile_categories') + + # Removing M2M table for field overlays on 'UserProfile' + db.delete_table('djeuscan_userprofile_overlays') + + # Deleting model 'Category' + db.delete_table('djeuscan_category') + + # Deleting model 'Overlay' + db.delete_table('djeuscan_overlay') + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'djeuscan.category': { + 'Meta': {'object_name': 'Category'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.categorylog': { + 'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.euscanresult': { + 'Meta': {'object_name': 'EuscanResult'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}) + }, + 'djeuscan.herd': { + 'Meta': {'object_name': 'Herd'}, + 'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'}) + }, + 'djeuscan.herdlog': { + 'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.log': { + 'Meta': {'object_name': 'Log'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'djeuscan.maintainer': { + 'Meta': {'object_name': 'Maintainer'}, + 'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.maintainerlog': { + 'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}) + }, + 'djeuscan.overlay': { + 'Meta': {'object_name': 'Overlay'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.package': { + 'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}), + 'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}), + 'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.problemreport': { + 'Meta': {'object_name': 'ProblemReport'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Version']", 'null': 'True', 'blank': 'True'}) + }, + 'djeuscan.refreshpackagequery': { + 'Meta': {'object_name': 'RefreshPackageQuery'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'symmetrical': 'False'}) + }, + 'djeuscan.userprofile': { + 'Meta': {'object_name': 'UserProfile'}, + 'categories': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Category']", 'symmetrical': 'False'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'}), + 'overlays': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Overlay']", 'symmetrical': 'False'}), + 'packages': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Package']", 'symmetrical': 'False'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'}) + }, + 'djeuscan.version': { + 'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'}, + 'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'ebuild_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'metadata_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.versionlog': { + 'Meta': {'object_name': 'VersionLog'}, + 'action': ('django.db.models.fields.IntegerField', [], {}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.worldlog': { + 'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + } + } + + complete_apps = ['djeuscan'] \ No newline at end of file diff --git a/euscanwww/djeuscan/migrations/0017_auto__add_unique_overlay_name__add_unique_category_name.py b/euscanwww/djeuscan/migrations/0017_auto__add_unique_overlay_name__add_unique_category_name.py new file mode 100644 index 0000000..7d95034 --- /dev/null +++ b/euscanwww/djeuscan/migrations/0017_auto__add_unique_overlay_name__add_unique_category_name.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding unique constraint on 'Overlay', fields ['name'] + db.create_unique('djeuscan_overlay', ['name']) + + # Adding unique constraint on 'Category', fields ['name'] + db.create_unique('djeuscan_category', ['name']) + + def backwards(self, orm): + # Removing unique constraint on 'Category', fields ['name'] + db.delete_unique('djeuscan_category', ['name']) + + # Removing unique constraint on 'Overlay', fields ['name'] + db.delete_unique('djeuscan_overlay', ['name']) + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'djeuscan.category': { + 'Meta': {'object_name': 'Category'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}) + }, + 'djeuscan.categorylog': { + 'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.euscanresult': { + 'Meta': {'object_name': 'EuscanResult'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}) + }, + 'djeuscan.herd': { + 'Meta': {'object_name': 'Herd'}, + 'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'}) + }, + 'djeuscan.herdlog': { + 'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.log': { + 'Meta': {'object_name': 'Log'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'djeuscan.maintainer': { + 'Meta': {'object_name': 'Maintainer'}, + 'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.maintainerlog': { + 'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}) + }, + 'djeuscan.overlay': { + 'Meta': {'object_name': 'Overlay'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}) + }, + 'djeuscan.package': { + 'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}), + 'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}), + 'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.problemreport': { + 'Meta': {'object_name': 'ProblemReport'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Version']", 'null': 'True', 'blank': 'True'}) + }, + 'djeuscan.refreshpackagequery': { + 'Meta': {'object_name': 'RefreshPackageQuery'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'symmetrical': 'False'}) + }, + 'djeuscan.userprofile': { + 'Meta': {'object_name': 'UserProfile'}, + 'categories': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Category']", 'symmetrical': 'False'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'}), + 'overlays': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Overlay']", 'symmetrical': 'False'}), + 'packages': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Package']", 'symmetrical': 'False'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'}) + }, + 'djeuscan.version': { + 'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'}, + 'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'ebuild_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'metadata_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.versionlog': { + 'Meta': {'object_name': 'VersionLog'}, + 'action': ('django.db.models.fields.IntegerField', [], {}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.worldlog': { + 'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + } + } + + complete_apps = ['djeuscan'] \ No newline at end of file diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py index 6626a41..86f79bc 100644 --- a/euscanwww/djeuscan/models.py +++ b/euscanwww/djeuscan/models.py @@ -231,6 +231,31 @@ class EuscanResult(models.Model): ) +class Category(models.Model): + name = models.CharField(max_length=128, validators=[validate_category], + unique=True) + + def __unicode__(self): + return self.name + + +class Overlay(models.Model): + name = models.CharField(max_length=128, validators=[validate_name], + unique=True) + + def __unicode__(self): + return self.name + + +class UserProfile(models.Model): + user = models.OneToOneField(User) + herds = models.ManyToManyField(Herd) + maintainers = models.ManyToManyField(Maintainer) + packages = models.ManyToManyField(Package) + categories = models.ManyToManyField(Category) + overlays = models.ManyToManyField(Overlay) + + class Log(models.Model): """ Model used for keeping data for charts @@ -303,61 +328,6 @@ class RefreshPackageQuery(models.Model): return u'[%d] %s' % (self.priority, self.package) -class HerdAssociation(models.Model): - user = models.ForeignKey(User) - herd = models.ForeignKey(Herd) - - class Meta: - unique_together = ['user', 'herd'] - - def __unicode__(self): - return u'[%s] %s' % (self.user, self.herd) - - -class MaintainerAssociation(models.Model): - user = models.ForeignKey(User) - maintainer = models.ForeignKey(Maintainer) - - class Meta: - unique_together = ['user', 'maintainer'] - - def __unicode__(self): - return u'[%s] %s' % (self.user, self.maintainer) - - -class PackageAssociation(models.Model): - user = models.ForeignKey(User) - package = models.ForeignKey(Package) - - class Meta: - unique_together = ['user', 'package'] - - def __unicode__(self): - return u'[%s] %s' % (self.user, self.package) - - -class CategoryAssociation(models.Model): - user = models.ForeignKey(User) - category = models.CharField(max_length=128, validators=[validate_category]) - - class Meta: - unique_together = ['user', 'category'] - - def __unicode__(self): - return u'[%s] %s' % (self.user, self.category) - - -class OverlayAssociation(models.Model): - user = models.ForeignKey(User) - overlay = models.CharField(max_length=128, validators=[validate_name]) - - class Meta: - unique_together = ['user', 'overlay'] - - def __unicode__(self): - return u'[%s] %s' % (self.user, self.category) - - class ProblemReport(models.Model): package = models.ForeignKey(Package) version = models.ForeignKey(Version, null=True, blank=True) diff --git a/euscanwww/djeuscan/processing/misc/update_counters.py b/euscanwww/djeuscan/processing/misc/update_counters.py index 09ba1db..02d9b4b 100644 --- a/euscanwww/djeuscan/processing/misc/update_counters.py +++ b/euscanwww/djeuscan/processing/misc/update_counters.py @@ -1,8 +1,8 @@ from django.db.transaction import commit_on_success from django.utils import timezone -from djeuscan.models import Package, Herd, Maintainer, Version -from djeuscan.models import HerdLog, MaintainerLog, CategoryLog, WorldLog +from djeuscan.models import Package, Herd, Maintainer, Version, HerdLog, \ + MaintainerLog, CategoryLog, WorldLog, Category, Overlay from djeuscan import charts from djeuscan.processing import FakeLogger @@ -48,6 +48,19 @@ def update_counters(fast=False, nolog=False, logger=None): wlog = None + # Populate Category and Overlay + for cat in Package.objects.values('category').distinct(): + obj, created = Category.objects.get_or_create(name=cat["category"]) + if created: + logger.info("+ [c] %s", cat["category"]) + + for overlay in Version.objects.values('overlay').distinct(): + if not overlay["overlay"]: + continue + obj, created = Overlay.objects.get_or_create(name=overlay["overlay"]) + if created: + logger.info("+ [o] %s", overlay["overlay"]) + if not nolog: wlog = WorldLog() wlog.datetime = now @@ -157,17 +170,17 @@ def update_counters(fast=False, nolog=False, logger=None): return for clog in categories.values(): - logger.info('+ [cl] %s' % clog) + logger.info('+ [cl] %s', clog) charts.rrd_update('category-%s' % clog.category, now, clog) clog.save() for hlog in herds.values(): - logger.info('+ [hl] %s' % hlog) + logger.info('+ [hl] %s', hlog) charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog) hlog.save() for mlog in maintainers.values(): - logger.info('+ [ml] %s' % mlog) + logger.info('+ [ml] %s', mlog) charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog) mlog.save() diff --git a/euscanwww/djeuscan/tests/euscan_factory.py b/euscanwww/djeuscan/tests/euscan_factory.py index 114e3a4..db53a32 100644 --- a/euscanwww/djeuscan/tests/euscan_factory.py +++ b/euscanwww/djeuscan/tests/euscan_factory.py @@ -4,10 +4,12 @@ from datetime import datetime from collections import defaultdict from django.contrib.auth.models import User +from django.db import IntegrityError import factory -from djeuscan.models import Herd, Maintainer, Package, Version, EuscanResult +from djeuscan.models import Herd, Maintainer, Package, Version, EuscanResult, \ + Category, Overlay class UserFactory(factory.Factory): @@ -36,6 +38,18 @@ class MaintainerFactory(factory.Factory): email = factory.LazyAttribute(lambda a: "%s@example.com" % a.name) +class CategoryFactory(factory.Factory): + FACTORY_FOR = Category + + name = factory.LazyAttribute(lambda a: random_string()) + + +class OverlayFactory(factory.Factory): + FACTORY_FOR = Overlay + + name = factory.LazyAttribute(lambda a: random_string()) + + class PackageFactory(factory.Factory): FACTORY_FOR = Package @@ -46,6 +60,16 @@ class PackageFactory(factory.Factory): description = "This is a test package" homepage = "http://testpackage.com" + @classmethod + def _prepare(cls, create, **kwargs): + package = super(PackageFactory, cls)._prepare(create, **kwargs) + category = kwargs.pop('category', None) + if create: + CategoryFactory.create(name=category) + else: + CategoryFactory.build(name=category) + return package + class VersionFactory(factory.Factory): FACTORY_FOR = Version @@ -59,6 +83,19 @@ class VersionFactory(factory.Factory): urls = "http://packageurl.com" alive = True + @classmethod + def _prepare(cls, create, **kwargs): + version = super(VersionFactory, cls)._prepare(create, **kwargs) + overlay = kwargs.pop('overlay', None) + try: + if create: + OverlayFactory.create(name=overlay) + else: + OverlayFactory.build(name=overlay) + except IntegrityError: + pass + return version + class EuscanResultFactory(factory.Factory): FACTORY_FOR = EuscanResult diff --git a/euscanwww/djeuscan/tests/test_views.py b/euscanwww/djeuscan/tests/test_views.py index 4d9ce4b..e9a7877 100644 --- a/euscanwww/djeuscan/tests/test_views.py +++ b/euscanwww/djeuscan/tests/test_views.py @@ -5,13 +5,12 @@ try: except ImportError: from bs4 import BeautifulSoup +from djeuscan.helpers import get_profile + from djeuscan.tests import SystemTestCase from djeuscan.tests.euscan_factory import PackageFactory, setup_maintainers, \ setup_herds, setup_categories, setup_overlays -from djeuscan.models import PackageAssociation, CategoryAssociation, \ - HerdAssociation, MaintainerAssociation - class PagesTest(SystemTestCase): """ @@ -50,11 +49,10 @@ class PackageTests(SystemTestCase): self.assertEqual(response.status_code, 200) def test_favourite(self): - self.assertEqual(PackageAssociation.objects.count(), 0) - response = self.get("package", category=self.package.category, package=self.package.name) self.assertEqual(response.status_code, 200) + self.assertNotIn("Watch", response.content) with self.login(): @@ -62,11 +60,14 @@ class PackageTests(SystemTestCase): package=self.package.name) self.assertEqual(response.status_code, 200) + user = response.context["user"] + self.assertEquals(get_profile(user).categories.count(), 0) + self.assertIn("Watch", response.content) self.post("favourite_package", category=self.package.category, package=self.package.name) - self.assertEqual(PackageAssociation.objects.count(), 1) + self.assertEquals(get_profile(user).packages.count(), 1) response = self.get("accounts_packages") self.assertEqual(response.status_code, 200) @@ -75,7 +76,7 @@ class PackageTests(SystemTestCase): self.post("unfavourite_package", category=self.package.category, package=self.package.name) - self.assertEqual(PackageAssociation.objects.count(), 0) + self.assertEquals(get_profile(user).packages.count(), 0) class SectionTests(SystemTestCase): @@ -118,9 +119,9 @@ class CategoriesTests(SectionTests): def test_favourite(self): category = self.categories[0] - self.assertEqual(CategoryAssociation.objects.count(), 0) response = self.get("category", category=category) + self.assertEqual(response.status_code, 200) self.assertNotIn("Watch", response.content) @@ -128,10 +129,13 @@ class CategoriesTests(SectionTests): response = self.get("category", category=category) self.assertEqual(response.status_code, 200) + user = response.context["user"] + self.assertEquals(get_profile(user).categories.count(), 0) + self.assertIn("Watch", response.content) self.post("favourite_category", category=category) - self.assertEqual(CategoryAssociation.objects.count(), 1) + self.assertEquals(get_profile(user).categories.count(), 1) response = self.get("accounts_categories") self.assertEqual(response.status_code, 200) @@ -139,7 +143,8 @@ class CategoriesTests(SectionTests): self._check_table(response, [category]) self.post("unfavourite_category", category=category) - self.assertEqual(CategoryAssociation.objects.count(), 0) + + self.assertEquals(get_profile(user).categories.count(), 0) class HerdsTests(SectionTests): @@ -167,7 +172,6 @@ class HerdsTests(SectionTests): def test_favourite(self): herd = self.herds[0] - self.assertEqual(HerdAssociation.objects.count(), 0) response = self.get("herd", herd=herd.herd) self.assertEqual(response.status_code, 200) @@ -177,10 +181,13 @@ class HerdsTests(SectionTests): response = self.get("herd", herd=herd.herd) self.assertEqual(response.status_code, 200) + user = response.context["user"] + self.assertEquals(get_profile(user).herds.count(), 0) + self.assertIn("Watch", response.content) self.post("favourite_herd", herd=herd.herd) - self.assertEqual(HerdAssociation.objects.count(), 1) + self.assertEquals(get_profile(user).herds.count(), 1) response = self.get("accounts_herds") self.assertEqual(response.status_code, 200) @@ -188,7 +195,7 @@ class HerdsTests(SectionTests): self._check_table(response, [herd], attr="herd") self.post("unfavourite_herd", herd=herd.herd) - self.assertEqual(HerdAssociation.objects.count(), 0) + self.assertEquals(get_profile(user).herds.count(), 0) class MaintainersTests(SectionTests): @@ -216,7 +223,6 @@ class MaintainersTests(SectionTests): def test_favourite(self): maintainer = self.maintainers[0] - self.assertEqual(MaintainerAssociation.objects.count(), 0) response = self.get("maintainer", maintainer_id=maintainer.pk) self.assertEqual(response.status_code, 200) @@ -226,10 +232,13 @@ class MaintainersTests(SectionTests): response = self.get("maintainer", maintainer_id=maintainer.pk) self.assertEqual(response.status_code, 200) + user = response.context["user"] + self.assertEquals(get_profile(user).maintainers.count(), 0) + self.assertIn("Watch", response.content) self.post("favourite_maintainer", maintainer_id=maintainer.pk) - self.assertEqual(MaintainerAssociation.objects.count(), 1) + self.assertEquals(get_profile(user).maintainers.count(), 1) response = self.get("accounts_maintainers") self.assertEqual(response.status_code, 200) @@ -237,7 +246,8 @@ class MaintainersTests(SectionTests): self._check_table(response, [maintainer], attr="name") self.post("unfavourite_maintainer", maintainer_id=maintainer.pk) - self.assertEqual(MaintainerAssociation.objects.count(), 0) + + self.assertEquals(get_profile(user).maintainers.count(), 0) class OverlayTests(SectionTests): diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py index 68b3d07..89460f4 100644 --- a/euscanwww/djeuscan/views.py +++ b/euscanwww/djeuscan/views.py @@ -8,12 +8,10 @@ from django.shortcuts import get_object_or_404 from django.contrib.auth.decorators import login_required from django.views.decorators.http import require_POST -from djeuscan.helpers import version_key, packages_from_names, \ - get_account_categories, get_account_herds, get_account_maintainers, \ - get_account_packages +from djeuscan.helpers import version_key, packages_from_names, get_profile, \ + get_account_categories, get_account_herds, get_account_maintainers from djeuscan.models import Version, Package, Herd, Maintainer, EuscanResult, \ - VersionLog, RefreshPackageQuery, HerdAssociation, MaintainerAssociation, \ - CategoryAssociation, PackageAssociation, OverlayAssociation, ProblemReport + VersionLog, RefreshPackageQuery, ProblemReport, Category, Overlay from djeuscan.forms import WorldForm, PackagesForm, ProblemReportForm from djeuscan.tasks import admin_tasks from djeuscan import charts @@ -68,12 +66,8 @@ def category(request, category): favourited = False if request.user.is_authenticated(): - try: - CategoryAssociation.objects.get(user=request.user, - category=category) - except CategoryAssociation.DoesNotExist: - pass - else: + if Category.objects.get(name=category) in \ + get_profile(request.user).categories.all(): favourited = True return {'category': category, 'packages': packages, 'last_scan': last_scan, @@ -104,11 +98,7 @@ def herd(request, herd): favourited = False if request.user.is_authenticated(): - try: - HerdAssociation.objects.get(user=request.user, herd=herd) - except HerdAssociation.DoesNotExist: - pass - else: + if herd in get_profile(request.user).herds.all(): favourited = True return {'herd': herd, 'packages': packages, "last_scan": last_scan, @@ -140,12 +130,7 @@ def maintainer(request, maintainer_id): favourited = False if request.user.is_authenticated(): - try: - MaintainerAssociation.objects.get(user=request.user, - maintainer=maintainer) - except MaintainerAssociation.DoesNotExist: - pass - else: + if maintainer in get_profile(request.user).maintainers.all(): favourited = True return {'maintainer': maintainer, 'packages': packages, @@ -177,11 +162,8 @@ def overlay(request, overlay): favourited = False if request.user.is_authenticated(): - try: - OverlayAssociation.objects.get(user=request.user, overlay=overlay) - except OverlayAssociation.DoesNotExist: - pass - else: + if Overlay.objects.get(name=overlay) in \ + get_profile(request.user).overlays.all(): favourited = True return {'overlay': overlay, 'packages': packages, 'last_scan': last_scan, @@ -209,11 +191,7 @@ def package(request, category, package): favourited = False if request.user.is_authenticated(): - try: - PackageAssociation.objects.get(user=request.user, package=package) - except PackageAssociation.DoesNotExist: - pass - else: + if package in get_profile(request.user).packages.all(): favourited = True try: @@ -226,8 +204,8 @@ def package(request, category, package): 'package': package, 'packaged': packaged, 'upstream': upstream, - 'log': log.messages(), 'vlog': vlog, + 'log': log, 'msg': log.messages() if log else "", 'last_scan': last_scan, 'favourited': favourited, @@ -397,9 +375,10 @@ def refresh_package(request, category, package): @login_required @render_to('euscan/accounts/index.html') def accounts_index(request): + user = request.user upstream_k = lambda c: c["n_versions"] - c["n_packaged"] - categories = sorted(get_account_categories(request.user), + categories = sorted(get_account_categories(user), key=upstream_k, reverse=True) c_upstream = sum([c["n_versions"] - c["n_packaged"] for c in categories]) herds = sorted(get_account_herds(request.user), @@ -408,7 +387,7 @@ def accounts_index(request): maintainers = sorted(get_account_maintainers(request.user), key=upstream_k, reverse=True) m_upstream = sum([c["n_versions"] - c["n_packaged"] for c in maintainers]) - packages = sorted(get_account_packages(request.user), + packages = sorted(get_profile(user).packages.all(), key=lambda p: p.n_versions - p.n_packaged, reverse=True) p_upstream = sum([c.n_versions - c.n_packaged for c in packages]) return { @@ -440,14 +419,13 @@ def accounts_maintainers(request): @login_required @render_to('euscan/accounts/packages.html') def accounts_packages(request): - return {"packages": get_account_packages(request.user)} + return {"packages": get_profile(request.user).packages.all()} @login_required @render_to('euscan/accounts/overlays.html') def accounts_overlays(request): - overlays = [obj.overlay for obj in - OverlayAssociation.objects.filter(user=request.user)] + overlays = [obj.name for obj in get_profile(request.user).overlays.all()] return {"overlays": overlays} @@ -456,10 +434,8 @@ def accounts_overlays(request): @ajax_request def favourite_package(request, category, package): obj = get_object_or_404(Package, category=category, name=package) - _, created = PackageAssociation.objects.get_or_create( - user=request.user, package=obj - ) - return {"success": created} + get_profile(request.user).packages.add(obj) + return {"success": True} @login_required @@ -467,10 +443,7 @@ def favourite_package(request, category, package): @ajax_request def unfavourite_package(request, category, package): package = get_object_or_404(Package, category=category, name=package) - obj = get_object_or_404( - PackageAssociation, package=package, user=request.user - ) - obj.delete() + get_profile(request.user).packages.remove(package) return {"success": True} @@ -479,10 +452,8 @@ def unfavourite_package(request, category, package): @ajax_request def favourite_herd(request, herd): obj = get_object_or_404(Herd, herd=herd) - _, created = HerdAssociation.objects.get_or_create( - user=request.user, herd=obj - ) - return {"success": created} + get_profile(request.user).herds.add(obj) + return {"success": True} @login_required @@ -490,10 +461,7 @@ def favourite_herd(request, herd): @ajax_request def unfavourite_herd(request, herd): herd = get_object_or_404(Herd, herd=herd) - obj = get_object_or_404( - HerdAssociation, herd=herd, user=request.user - ) - obj.delete() + get_profile(request.user).herds.remove(herd) return {"success": True} @@ -502,10 +470,8 @@ def unfavourite_herd(request, herd): @ajax_request def favourite_maintainer(request, maintainer_id): obj = get_object_or_404(Maintainer, pk=maintainer_id) - _, created = MaintainerAssociation.objects.get_or_create( - user=request.user, maintainer=obj - ) - return {"success": created} + get_profile(request.user).maintainers.add(obj) + return {"success": True} @login_required @@ -513,10 +479,7 @@ def favourite_maintainer(request, maintainer_id): @ajax_request def unfavourite_maintainer(request, maintainer_id): maintainer = get_object_or_404(Maintainer, pk=maintainer_id) - obj = get_object_or_404( - MaintainerAssociation, maintainer=maintainer, user=request.user - ) - obj.delete() + get_profile(request.user).maintainers.remove(maintainer) return {"success": True} @@ -524,25 +487,17 @@ def unfavourite_maintainer(request, maintainer_id): @require_POST @ajax_request def favourite_category(request, category): - packages = Package.objects.for_category(category, last_versions=True) - - if not packages: - raise Http404 - - _, created = CategoryAssociation.objects.get_or_create( - user=request.user, category=category - ) - return {"success": created} + obj = Category.objects.get(name=category) + get_profile(request.user).categories.add(obj) + return {"success": True} @login_required @require_POST @ajax_request def unfavourite_category(request, category): - obj = get_object_or_404( - CategoryAssociation, user=request.user, category=category - ) - obj.delete() + obj = Category.objects.get(name=category) + get_profile(request.user).categories.remove(obj) return {"success": True} @@ -550,22 +505,15 @@ def unfavourite_category(request, category): @require_POST @ajax_request def favourite_overlay(request, overlay): - packages = Package.objects.for_overlay(overlay) - if not packages: - raise Http404 - - _, created = OverlayAssociation.objects.get_or_create( - user=request.user, overlay=overlay - ) - return {"success": created} + obj = Category.objects.get(name=overlay) + get_profile(request.user).overlays.add(obj) + return {"success": True} @login_required @require_POST @ajax_request def unfavourite_overlay(request, overlay): - obj = get_object_or_404( - OverlayAssociation, user=request.user, overlay=overlay - ) - obj.delete() + obj = Category.objects.get(name=overlay) + get_profile(request.user).overlays.remove(obj) return {"success": True} diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py index fc6b668..d70c99c 100644 --- a/euscanwww/euscanwww/settings.py +++ b/euscanwww/euscanwww/settings.py @@ -261,3 +261,5 @@ except ImportError, ex: os.environ['ROOT'] = PORTAGE_ROOT os.environ['PORTAGE_CONFIGROOT'] = PORTAGE_CONFIGROOT os.environ['EIX_CACHEFILE'] = EIX_CACHEFILE + +AUTH_PROFILE_MODULE = 'djeuscan.UserProfile' diff --git a/euscanwww/runtests.py b/euscanwww/runtests.py index 3721076..a436a27 100644 --- a/euscanwww/runtests.py +++ b/euscanwww/runtests.py @@ -29,6 +29,7 @@ settings.configure( USE_TZ=True, TASKS_CONCURRENTLY=8, TASKS_SUBTASK_PACKAGES=32, + AUTH_PROFILE_MODULE="djeuscan.UserProfile" ) From 1835327d5c41cf4508d04827f2eb2435cc3fb7f4 Mon Sep 17 00:00:00 2001 From: volpino Date: Thu, 26 Jul 2012 14:50:47 +0200 Subject: [PATCH 15/17] euscan: Fixed typo Signed-off-by: volpino --- pym/euscan/handlers/url/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/euscan/handlers/url/generic.py b/pym/euscan/handlers/url/generic.py index 38cfa10..1f43a7a 100644 --- a/pym/euscan/handlers/url/generic.py +++ b/pym/euscan/handlers/url/generic.py @@ -143,7 +143,7 @@ def scan(pkg, url): ret = scan_directory_recursive(cp, ver, rev, "", steps, url) if not ret: - brute_force(pkg, url) + ret = brute_force(pkg, url) return ret From ad0c9462301a735b54b7968648e2d6afbc137516 Mon Sep 17 00:00:00 2001 From: volpino Date: Thu, 26 Jul 2012 15:50:30 +0200 Subject: [PATCH 16/17] euscan: patch_metadata regex fix to match bz2 sources Signed-off-by: volpino --- bin/euscan_patch_metadata | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/euscan_patch_metadata b/bin/euscan_patch_metadata index 617ca3b..2f1a834 100755 --- a/bin/euscan_patch_metadata +++ b/bin/euscan_patch_metadata @@ -117,7 +117,7 @@ def get_deb_url(name): content = opened.read() for link in BeautifulSoup(content, parseOnlyThese=SoupStrainer("a")): - if re.match("[^\s]+\.debian\.tar\.gz", link.text): + if re.match("[^\s]+\.debian\.tar\.(?:gz|bz2)", link.text): deb_url = link["href"] deb_type = "source" break From 35603fd704fa86589fe3df4480182b0cec852559 Mon Sep 17 00:00:00 2001 From: volpino Date: Thu, 26 Jul 2012 17:46:53 +0200 Subject: [PATCH 17/17] euscan: Added remote-id handler Signed-off-by: volpino --- pym/euscan/handlers/package/remote_id.py | 43 ++++++ pym/euscan/handlers/url/cpan.py | 23 ++- pym/euscan/handlers/url/github.py | 5 +- pym/euscan/handlers/url/pypi.py | 13 +- .../handlers/url/{rubygem.py => rubygems.py} | 22 +-- pym/euscan/handlers/watch.py | 143 ------------------ 6 files changed, 85 insertions(+), 164 deletions(-) create mode 100644 pym/euscan/handlers/package/remote_id.py rename pym/euscan/handlers/url/{rubygem.py => rubygems.py} (82%) delete mode 100644 pym/euscan/handlers/watch.py diff --git a/pym/euscan/handlers/package/remote_id.py b/pym/euscan/handlers/package/remote_id.py new file mode 100644 index 0000000..b95f856 --- /dev/null +++ b/pym/euscan/handlers/package/remote_id.py @@ -0,0 +1,43 @@ +from euscan.handlers.url import handlers +from euscan import output + +PRIORITY = 100 + +HANDLER_NAME = "remote_id" +CONFIDENCE = 100.0 + + +url_handlers = {handler.HANDLER_NAME: handler for handler in handlers} + + +def can_handle(pkg): + # Return True if there's at least one remote-id that can be + # handled by euscan + try: + remoteids = pkg.metadata.upstream()[0].upstream_remoteids() + except IndexError: + pass + else: + if len(remoteids) > 0: + for remote_value, remote_type in remoteids: + if remote_type in url_handlers: + return True + return False + + +def scan(pkg): + output.einfo("Using remote-id data") + + ret = [] + + remoteids = pkg.metadata.upstream()[0].upstream_remoteids() + for remote_value, remote_type in remoteids: + if remote_type in url_handlers: + remote_data = remote_value.split("/") + scan_remote = getattr( + url_handlers[remote_type], "scan_remote", None + ) + if scan_remote: + for url, pv in scan_remote(pkg, remote_data): + ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) + return ret diff --git a/pym/euscan/handlers/url/cpan.py b/pym/euscan/handlers/url/cpan.py index 0f587e1..0721324 100644 --- a/pym/euscan/handlers/url/cpan.py +++ b/pym/euscan/handlers/url/cpan.py @@ -81,12 +81,22 @@ def cpan_vercmp(cp, a, b): def scan(pkg, url): cp, ver, rev = portage.pkgsplit(pkg.cpv) - pkg = guess_package(cp, url) + remote_pkg = guess_package(cp, url) - orig_url = url - url = 'http://search.cpan.org/api/dist/%s' % pkg + output.einfo("Using CPAN API: %s", remote_pkg) - output.einfo("Using: " + url) + result = scan_remote(pkg, [remote_pkg]) + + ret = [] + for url, pv in result: + ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) + return ret + + +def scan_remote(pkg, remote_data): + remote_pkg = remote_data[0] + url = 'http://search.cpan.org/api/dist/%s' % remote_pkg + cp, ver, rev = portage.pkgsplit(pkg.cpv) try: fp = helpers.urlopen(url) @@ -125,9 +135,6 @@ def scan(pkg, url): version['archive'] ) - if url == orig_url: - continue - - ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) + ret.append((url, pv)) return ret diff --git a/pym/euscan/handlers/url/github.py b/pym/euscan/handlers/url/github.py index 080a559..e4ebe10 100644 --- a/pym/euscan/handlers/url/github.py +++ b/pym/euscan/handlers/url/github.py @@ -44,6 +44,7 @@ def scan(pkg, url): (user, project)) dls = json.load(dlreq) + ret = [] for dl in dls: m = fnre.match(dl['name']) @@ -51,4 +52,6 @@ def scan(pkg, url): pv = helpers.gentoo_mangle_version(m.group(1)) if helpers.version_filtered(cp, ver, pv): continue - yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE) + + ret.append((dl['html_url'], pv, HANDLER_NAME, CONFIDENCE)) + return ret diff --git a/pym/euscan/handlers/url/pypi.py b/pym/euscan/handlers/url/pypi.py index 8ed8021..82251e6 100644 --- a/pym/euscan/handlers/url/pypi.py +++ b/pym/euscan/handlers/url/pypi.py @@ -29,6 +29,15 @@ def scan(pkg, url): package = guess_package(pkg.cpv, url) + ret = [] + for urls, pv in scan_remote(pkg, [package]): + ret.append((urls, pv, HANDLER_NAME, CONFIDENCE)) + return ret + + +def scan_remote(pkg, remote_data): + package = remote_data[0] + output.einfo("Using PyPi XMLRPC: " + package) client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi') @@ -42,13 +51,11 @@ def scan(pkg, url): cp, ver, rev = portage.pkgsplit(pkg.cpv) ret = [] - for up_pv in versions: pv = helpers.gentoo_mangle_version(up_pv) if helpers.version_filtered(cp, ver, pv): continue urls = client.release_urls(package, up_pv) urls = " ".join([infos['url'] for infos in urls]) - ret.append((urls, pv, HANDLER_NAME, CONFIDENCE)) - + ret.append((urls, pv)) return ret diff --git a/pym/euscan/handlers/url/rubygem.py b/pym/euscan/handlers/url/rubygems.py similarity index 82% rename from pym/euscan/handlers/url/rubygem.py rename to pym/euscan/handlers/url/rubygems.py index 950e81b..a3021f0 100644 --- a/pym/euscan/handlers/url/rubygem.py +++ b/pym/euscan/handlers/url/rubygems.py @@ -5,7 +5,7 @@ import urllib2 from euscan import helpers, output -HANDLER_NAME = "rubygem" +HANDLER_NAME = "rubygems" CONFIDENCE = 100.0 PRIORITY = 90 @@ -33,14 +33,23 @@ def scan(pkg, url): 'http://guides.rubygems.org/rubygems-org-api/#gemversion' gem = guess_gem(pkg.cpv, url) + if not gem: output.eerror("Can't guess gem name using %s and %s" % \ (pkg.cpv, url)) return [] - url = 'http://rubygems.org/api/v1/versions/%s.json' % gem + output.einfo("Using RubyGem API: %s" % gem) - output.einfo("Using: " + url) + ret = [] + for url, pv in scan_remote(pkg, [gem]): + ret.append(url, pv, HANDLER_NAME, CONFIDENCE) + return ret + + +def scan_remote(pkg, remote_data): + gem = remote_data[0] + url = 'http://rubygems.org/api/v1/versions/%s.json' % gem try: fp = helpers.urlopen(url) @@ -55,19 +64,14 @@ def scan(pkg, url): data = fp.read() versions = json.loads(data) - if not versions: - return [] - cp, ver, rev = portage.pkgsplit(pkg.cpv) ret = [] - for version in versions: up_pv = version['number'] pv = helpers.gentoo_mangle_version(up_pv) if helpers.version_filtered(cp, ver, pv): continue url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv) - ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) - + ret.append((url, pv)) return ret diff --git a/pym/euscan/handlers/watch.py b/pym/euscan/handlers/watch.py deleted file mode 100644 index a129281..0000000 --- a/pym/euscan/handlers/watch.py +++ /dev/null @@ -1,143 +0,0 @@ -import re -import urllib2 - -import portage - -from euscan.handlers import generic -from euscan import output, helpers - -PRIORITY = 100 - -HANDLER_NAME = "watch" -CONFIDENCE = 100.0 - - -is_pattern = r"\([^\/]+\)" - - -def can_handle(pkg, url): - try: - return pkg.metadata._xml_tree.find("upstream").find("watch") \ - is not None - except AttributeError: - return False - - -def parse_mangles(mangles, string): - for mangle in mangles: - # convert regex from perl format to python format - # there are some regex in this format: s/pattern/replacement/ - m = re.match(r"s/(.*[^\\])/(.*)/", mangle) - if not m: - # or in this format s|pattern|replacement| - m = re.match(r"s\|(.*[^\\])\|(.*)\|", mangle) - pattern, repl = m.groups() - repl = re.sub(r"\$(\d+)", r"\\\1", repl) - string = re.sub(pattern, repl, string) - return string - - -def clean_results(results, versionmangle, urlmangle): - ret = [] - - for path, version, _, _ in results: - version = parse_mangles(versionmangle, version) - path = parse_mangles(urlmangle, path) - ret.append((path, version, HANDLER_NAME, CONFIDENCE)) - - return ret - - -def parse_watch(pkg): - for watch_tag in pkg.metadata._xml_tree.find("upstream").findall("watch"): - try: - base, file_pattern = watch_tag.text.split(" ")[:2] - except ValueError: - base, file_pattern = watch_tag.text, None - - # the file pattern can be in the base url - pattern_regex = r"/([^/]*\([^/]*\)[^/]*)$" - match = re.search(pattern_regex, base) - if match: - file_pattern = match.group(1) - base = base.replace(file_pattern, "") - - # handle sf.net specially - base = base.replace( - "http://sf.net/", "http://qa.debian.org/watch/sf.php/" - ) - - vmangle = watch_tag.attrib.get("uversionmangle", None) or \ - watch_tag.attrib.get("versionmangle", None) - versionmangle = vmangle.split(";") if vmangle else [] - - umangle = watch_tag.attrib.get("downloadurlmangle", None) - urlmangle = umangle.split(";") if umangle else [] - - yield (base, file_pattern, versionmangle, urlmangle) - - -def handle_directory_patterns(base, file_pattern): - """ - Directory pattern matching - e.g.: base: ftp://ftp.nessus.org/pub/nessus/nessus-([\d\.]+)/src/ - file_pattern: nessus-core-([\d\.]+)\.tar\.gz - """ - splitted = base.split("/") - i = 0 - basedir = [] - for elem in splitted: - if re.search(is_pattern, elem): - break - basedir.append(elem) - i += 1 - basedir = "/".join(basedir) - directory_pattern = splitted[i] - final = "/".join(splitted[i + 1:]) - - try: - fp = helpers.urlopen(basedir) - except urllib2.URLError: - return [] - except IOError: - return [] - - if not fp: - return [] - - data = fp.read() - - if basedir.startswith("ftp://"): - scan_data = generic.scan_ftp(data, basedir, directory_pattern) - else: - scan_data = generic.scan_html(data, basedir, directory_pattern) - - return [("/".join((basedir, path, final)), file_pattern) - for _, path in scan_data] - - -def scan(pkg, url): - output.einfo("Using watch data") - - cp, ver, rev = portage.pkgsplit(pkg.cpv) - - results = [] - for base, file_pattern, versionmangle, urlmangle in parse_watch(pkg): - if not re.search(is_pattern, base): - steps = [(base, file_pattern)] - res = generic.scan_directory_recursive( - cp, ver, rev, "", steps, url - ) - else: - res = [] - for step in handle_directory_patterns(base, file_pattern): - res += generic.scan_directory_recursive( - cp, ver, rev, "", [step], url - ) - - results += clean_results(res, versionmangle, urlmangle) - return results - - -def brute_force(pkg, url): - return []