From 46c2ea6f70758c4df56c1bc9cb28e0a0d3637c9d Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Tue, 24 Jul 2012 08:33:19 +0200 Subject: [PATCH 01/14] euscan: use python-git for __version__ Signed-off-by: Corentin Chary --- bin/euscan | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bin/euscan b/bin/euscan index 8696848..42d63e8 100755 --- a/bin/euscan +++ b/bin/euscan @@ -5,21 +5,26 @@ Distributed under the terms of the GNU General Public License v2 """ from __future__ import print_function - +import os # Meta __author__ = "Corentin Chary (iksaif)" __email__ = "corentin.chary@gmail.com" -__version__ = "git" __productname__ = "euscan" __description__ = "A tool to detect new upstream releases." +if os.path.exists(os.path.join(os.path.dirname(__file__), "..", ".git")): + import git + repo = git.Repo(os.path.join(os.path.dirname(__file__))) + __version__ = "git-%s" % repo.head.commit +else: + __version__ = "git" + # Imports import sys -import os import getopt from errno import EINTR, EINVAL from httplib import HTTPConnection From 99054971aafdef31fe6c75801552d41b1aee57d3 Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Tue, 24 Jul 2012 08:37:31 +0200 Subject: [PATCH 02/14] euscan: drop fixed version dependencies Signed-off-by: Corentin Chary --- bin/euscan | 8 +------- setup.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/bin/euscan b/bin/euscan index 42d63e8..eb637d1 100755 --- a/bin/euscan +++ b/bin/euscan @@ -11,16 +11,10 @@ import os __author__ = "Corentin Chary (iksaif)" __email__ = "corentin.chary@gmail.com" +__version__ = "git" __productname__ = "euscan" __description__ = "A tool to detect new upstream releases." -if os.path.exists(os.path.join(os.path.dirname(__file__), "..", ".git")): - import git - repo = git.Repo(os.path.join(os.path.dirname(__file__))) - __version__ = "git-%s" % repo.head.commit -else: - __version__ = "git" - # Imports diff --git a/setup.py b/setup.py index 85856aa..9b7179f 100755 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ packages = [ ] tests_require = [ - 'factory-boy==1.1.3', + 'factory-boy>=1.1.3', ] setup( @@ -88,10 +88,13 @@ setup( ('master' if __version__ == '9999' else ('euscan-%s' % __version__)) ), install_requires=[ - 'Django==1.4', 'django-annoying==0.7.6', 'South==0.7.4', - 'django-piston==0.2.3', 'BeautifulSoup==3.2.1', 'matplotlib==1.1.0', - 'django-celery==3.0.1', 'django-registration==0.8', - 'python-ldap==2.4.10', 'django-auth-ldap==1.1', + # Command line utility + 'BeautifulSoup>=3.2.1', + # Web interface + 'Django>=1.4', 'django-annoying>=0.7.6', 'South>=0.7', + 'django-piston>=0.2.3', 'matplotlib>=1.1.0', + 'django-celery>=3.0.1', 'django-registration>=0.8', + 'python-ldap>=2.4.10', 'django-auth-ldap>=1.1', ], package_dir={'': 'pym'}, packages=packages, From 7cdd10f0283117d99ab81297d2c52829891d030e Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Tue, 24 Jul 2012 09:04:20 +0200 Subject: [PATCH 03/14] Revert "djeuscan: helpers.py is actually unused" This reverts commit be9d28c2557bdf09203bbc124b9fd434325ccc47. --- euscanwww/djeuscan/helpers.py | 108 ++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 euscanwww/djeuscan/helpers.py diff --git a/euscanwww/djeuscan/helpers.py b/euscanwww/djeuscan/helpers.py new file mode 100644 index 0000000..f7a6b82 --- /dev/null +++ b/euscanwww/djeuscan/helpers.py @@ -0,0 +1,108 @@ +""" +djeuscan.helpers +""" + +from distutils.version import StrictVersion, LooseVersion + + +def xint(i): + """ + Tries to cast to int, fallbacks to 0 + """ + try: + return int(i) + except Exception: + return 0 + + +def select_related_last_versions(queryset): + queryset = queryset.select_related( + 'last_version_gentoo', + 'last_version_overlay', + 'last_version_upstream' + ) + + +def version_key(version): + version = version.version + try: + return StrictVersion(version) + # in case of abnormal version number, fall back to LooseVersion + except ValueError: + return LooseVersion(version) + + +def packages_from_names(data): + """ + Returns a list of Package objects from a string of names + """ + + from djeuscan.models import Package + + packages = [] + data = data.replace("\r", "") + + for pkg in data.split('\n'): + if '/' in pkg: + cat, pkg = pkg.split('/') + packages.extend(Package.objects.filter(category=cat, name=pkg)) + else: + packages.extend(Package.objects.filter(name=pkg)) + return packages + + +def rename_fields(vqs, fields): + ret = [] + for n in vqs: + for tr in fields: + if tr[0] in n: + n[tr[1]] = n[tr[0]] + del n[tr[0]] + ret.append(n) + return ret + + +class catch_and_return(object): + def __init__(self, err, response): + self.err = err + self.response = response + + def __call__(self, fn): + def wrapper(*args, **kwargs): + try: + return fn(*args, **kwargs) + except self.err: + return self.response + return wrapper + + +def get_account_categories(user): + from djeuscan.models import Package, CategoryAssociation + + # TODO: This is quite ugly + category_names = [obj.category for obj in + CategoryAssociation.objects.filter(user=user)] + return [c for c in Package.objects.categories() + if c["category"] in category_names] + + +def get_account_herds(user): + from djeuscan.models import Package, HerdAssociation + + ids = [obj.herd.pk for obj in + HerdAssociation.objects.filter(user=user)] + return Package.objects.herds(ids=ids) + + +def get_account_maintainers(user): + from djeuscan.models import Package, MaintainerAssociation + + ids = [obj.maintainer.pk for obj in + MaintainerAssociation.objects.filter(user=user)] + return Package.objects.maintainers(ids=ids) + + +def get_account_packages(user): + from djeuscan.models import PackageAssociation + return [obj.package for obj in + PackageAssociation.objects.filter(user=user)] From 8146dc1f44946c4d894652b92f0848e97f28b7d7 Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Sat, 21 Jul 2012 07:36:06 +0200 Subject: [PATCH 04/14] djeuscan: helpers.py is actually unused Signed-off-by: Corentin Chary --- euscanwww/djeuscan/helpers.py | 108 ---------------------------------- 1 file changed, 108 deletions(-) delete mode 100644 euscanwww/djeuscan/helpers.py diff --git a/euscanwww/djeuscan/helpers.py b/euscanwww/djeuscan/helpers.py deleted file mode 100644 index f7a6b82..0000000 --- a/euscanwww/djeuscan/helpers.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -djeuscan.helpers -""" - -from distutils.version import StrictVersion, LooseVersion - - -def xint(i): - """ - Tries to cast to int, fallbacks to 0 - """ - try: - return int(i) - except Exception: - return 0 - - -def select_related_last_versions(queryset): - queryset = queryset.select_related( - 'last_version_gentoo', - 'last_version_overlay', - 'last_version_upstream' - ) - - -def version_key(version): - version = version.version - try: - return StrictVersion(version) - # in case of abnormal version number, fall back to LooseVersion - except ValueError: - return LooseVersion(version) - - -def packages_from_names(data): - """ - Returns a list of Package objects from a string of names - """ - - from djeuscan.models import Package - - packages = [] - data = data.replace("\r", "") - - for pkg in data.split('\n'): - if '/' in pkg: - cat, pkg = pkg.split('/') - packages.extend(Package.objects.filter(category=cat, name=pkg)) - else: - packages.extend(Package.objects.filter(name=pkg)) - return packages - - -def rename_fields(vqs, fields): - ret = [] - for n in vqs: - for tr in fields: - if tr[0] in n: - n[tr[1]] = n[tr[0]] - del n[tr[0]] - ret.append(n) - return ret - - -class catch_and_return(object): - def __init__(self, err, response): - self.err = err - self.response = response - - def __call__(self, fn): - def wrapper(*args, **kwargs): - try: - return fn(*args, **kwargs) - except self.err: - return self.response - return wrapper - - -def get_account_categories(user): - from djeuscan.models import Package, CategoryAssociation - - # TODO: This is quite ugly - category_names = [obj.category for obj in - CategoryAssociation.objects.filter(user=user)] - return [c for c in Package.objects.categories() - if c["category"] in category_names] - - -def get_account_herds(user): - from djeuscan.models import Package, HerdAssociation - - ids = [obj.herd.pk for obj in - HerdAssociation.objects.filter(user=user)] - return Package.objects.herds(ids=ids) - - -def get_account_maintainers(user): - from djeuscan.models import Package, MaintainerAssociation - - ids = [obj.maintainer.pk for obj in - MaintainerAssociation.objects.filter(user=user)] - return Package.objects.maintainers(ids=ids) - - -def get_account_packages(user): - from djeuscan.models import PackageAssociation - return [obj.package for obj in - PackageAssociation.objects.filter(user=user)] From 83cc025170fcb6f77753ab560a5849d07696725b Mon Sep 17 00:00:00 2001 From: volpino Date: Sun, 22 Jul 2012 10:26:40 +0200 Subject: [PATCH 05/14] euscan: euscan_patch_metadata --diff option Signed-off-by: volpino --- bin/euscan_patch_metadata | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/bin/euscan_patch_metadata b/bin/euscan_patch_metadata index 439105e..b5649ed 100755 --- a/bin/euscan_patch_metadata +++ b/bin/euscan_patch_metadata @@ -1,6 +1,7 @@ #!/usr/bin/env python import os +import sys import re import urllib from tempfile import mkstemp @@ -8,6 +9,7 @@ import tarfile import gzip import logging import shutil +import subprocess from gentoolkit.query import Query from BeautifulSoup import BeautifulSoup, SoupStrainer @@ -130,7 +132,7 @@ def get_deb_url(name): return deb_url, deb_type -def patch_metadata(metadata_path, watch_data): +def patch_metadata(metadata_path, watch_data, diff=False): watch_data = "\n".join([line for line in watch_data.split("\n") if not line.startswith("#")]) # comments watch_data = watch_data.replace("\\\n", "") # remove backslashes @@ -167,10 +169,20 @@ def patch_metadata(metadata_path, watch_data): (rindent, watch_tag, rindent) data = data.replace('', rep, 1) - print data + if not diff: + return data + else: + _, data_path = mkstemp() + with open(data_path, "w") as f: + f.write(data) + sub = subprocess.Popen(["diff", metadata_path, data_path], + stdout=subprocess.PIPE) + content = sub.stdout.read() + os.unlink(data_path) + return content -def process_package(query): +def process_package(query, diff=False): matches = Query(query).smart_find( in_installed=True, in_porttree=True, @@ -194,7 +206,7 @@ def process_package(query): if watch_data is None: logger.error(" No watch file found") else: - patch_metadata(metadata_path, watch_data) + return patch_metadata(metadata_path, watch_data, diff=diff) def main(): @@ -202,13 +214,17 @@ def main(): p = optparse.OptionParser( usage="usage: %prog [ [...]]", ) + p.add_option('-d', '--diff', action="store_true", dest="diff", + default=False, + help="Outputs a diff") opts, packages = p.parse_args() - logging.basicConfig(level=logging.INFO, format='%(message)s') + logging.basicConfig(stream=sys.stderr, level=logging.INFO, + format='%(message)s') for package in packages: logger.info("Processing %s..." % package) - process_package(package) + print process_package(package, opts.diff) if __name__ == "__main__": main() From cf8f9ae9cc95b043e10a9a1ac4b940582494152e Mon Sep 17 00:00:00 2001 From: volpino Date: Sun, 22 Jul 2012 10:34:13 +0200 Subject: [PATCH 06/14] euscanwww: From fixture to south data migration Signed-off-by: volpino --- euscanwww/djeuscan/fixtures/initial_data.json | 89 ----- .../0015_initial_celery_periodictasks.py | 304 ++++++++++++++++++ 2 files changed, 304 insertions(+), 89 deletions(-) delete mode 100644 euscanwww/djeuscan/fixtures/initial_data.json create mode 100644 euscanwww/djeuscan/migrations/0015_initial_celery_periodictasks.py diff --git a/euscanwww/djeuscan/fixtures/initial_data.json b/euscanwww/djeuscan/fixtures/initial_data.json deleted file mode 100644 index 250ab2f..0000000 --- a/euscanwww/djeuscan/fixtures/initial_data.json +++ /dev/null @@ -1,89 +0,0 @@ -[ - { - "pk": 1, - "model": "djcelery.intervalschedule", - "fields": { - "every": 1, - "period": "minutes" - } - }, - { - "pk": 2, - "model": "djcelery.intervalschedule", - "fields": { - "every": 1, - "period": "days" - } - }, - { - "pk": 3, - "model": "djcelery.intervalschedule", - "fields": { - "every": 7, - "period": "days" - } - }, - { - "pk": 2, - "model": "djcelery.periodictask", - "fields": { - "task": "djeuscan.tasks.consume_refresh_package_request", - "name": "User refresh queries", - "exchange": null, - "last_run_at": null, - "args": "[]", - "enabled": true, - "routing_key": null, - "crontab": null, - "interval": 1, - "queue": null, - "total_run_count": 0, - "expires": null, - "kwargs": "{}", - "date_changed": "2012-07-21T07:55:54.716Z", - "description": "" - } - }, - { - "pk": 3, - "model": "djcelery.periodictask", - "fields": { - "task": "djeuscan.tasks.update_portage", - "name": "Daily portage scan", - "exchange": null, - "last_run_at": null, - "args": "[]", - "enabled": true, - "routing_key": null, - "crontab": null, - "interval": 2, - "queue": null, - "total_run_count": 0, - "expires": null, - "kwargs": "{}", - "date_changed": "2012-07-21T07:56:25.235Z", - "description": "" - } - }, - { - "pk": 4, - "model": "djcelery.periodictask", - "fields": { - "task": "djeuscan.tasks.update_upstream", - "name": "Weekly upstream update", - "exchange": null, - "last_run_at": null, - "args": "[]", - "enabled": true, - "routing_key": null, - "crontab": null, - "interval": 3, - "queue": null, - "total_run_count": 0, - "expires": null, - "kwargs": "{}", - "date_changed": "2012-07-21T07:56:53.470Z", - "description": "" - } - } -] diff --git a/euscanwww/djeuscan/migrations/0015_initial_celery_periodictasks.py b/euscanwww/djeuscan/migrations/0015_initial_celery_periodictasks.py new file mode 100644 index 0000000..a5b3bed --- /dev/null +++ b/euscanwww/djeuscan/migrations/0015_initial_celery_periodictasks.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- +from south.v2 import DataMigration + + +class Migration(DataMigration): + + def forwards(self, orm): + every_minute = orm["djcelery.IntervalSchedule"].objects.create( + every=1, period="minutes" + ) + every_day = orm["djcelery.IntervalSchedule"].objects.create( + every=1, period="days" + ) + every_week = orm["djcelery.IntervalSchedule"].objects.create( + every=7, period="days" + ) + orm["djcelery.PeriodicTask"].objects.create( + name="Refresh package queries", + task="djeuscan.tasks.consume_refresh_package_request", + interval=every_minute + ) + orm["djcelery.PeriodicTask"].objects.create( + name="Daily portage update", + task="djeuscan.tasks.update_portage", + interval=every_day + ) + orm["djcelery.PeriodicTask"].objects.create( + name="Weekly upstream update", + task="djeuscan.tasks.update_upstream", + interval=every_week + ) + + def backwards(self, orm): + orm["djcelery.IntervalSchedule"].objects.all().delete() + orm["djcelery.PeriodicTask"].objects.all().delete() + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'djcelery.crontabschedule': { + 'Meta': {'object_name': 'CrontabSchedule'}, + 'day_of_month': ('django.db.models.fields.CharField', [], {'default': "'*'", 'max_length': '64'}), + 'day_of_week': ('django.db.models.fields.CharField', [], {'default': "'*'", 'max_length': '64'}), + 'hour': ('django.db.models.fields.CharField', [], {'default': "'*'", 'max_length': '64'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'minute': ('django.db.models.fields.CharField', [], {'default': "'*'", 'max_length': '64'}), + 'month_of_year': ('django.db.models.fields.CharField', [], {'default': "'*'", 'max_length': '64'}) + }, + 'djcelery.intervalschedule': { + 'Meta': {'object_name': 'IntervalSchedule'}, + 'every': ('django.db.models.fields.IntegerField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'period': ('django.db.models.fields.CharField', [], {'max_length': '24'}) + }, + 'djcelery.periodictask': { + 'Meta': {'object_name': 'PeriodicTask'}, + 'args': ('django.db.models.fields.TextField', [], {'default': "'[]'", 'blank': 'True'}), + 'crontab': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djcelery.CrontabSchedule']", 'null': 'True', 'blank': 'True'}), + 'date_changed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'exchange': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'expires': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interval': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djcelery.IntervalSchedule']", 'null': 'True', 'blank': 'True'}), + 'kwargs': ('django.db.models.fields.TextField', [], {'default': "'{}'", 'blank': 'True'}), + 'last_run_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'}), + 'queue': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'routing_key': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'task': ('django.db.models.fields.CharField', [], {'max_length': '200'}), + 'total_run_count': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}) + }, + 'djcelery.periodictasks': { + 'Meta': {'object_name': 'PeriodicTasks'}, + 'ident': ('django.db.models.fields.SmallIntegerField', [], {'default': '1', 'unique': 'True', 'primary_key': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {}) + }, + 'djcelery.taskmeta': { + 'Meta': {'object_name': 'TaskMeta', 'db_table': "'celery_taskmeta'"}, + 'date_done': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'meta': ('djcelery.picklefield.PickledObjectField', [], {'default': 'None', 'null': 'True'}), + 'result': ('djcelery.picklefield.PickledObjectField', [], {'default': 'None', 'null': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'PENDING'", 'max_length': '50'}), + 'task_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'traceback': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'djcelery.tasksetmeta': { + 'Meta': {'object_name': 'TaskSetMeta', 'db_table': "'celery_tasksetmeta'"}, + 'date_done': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'result': ('djcelery.picklefield.PickledObjectField', [], {}), + 'taskset_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'djcelery.taskstate': { + 'Meta': {'ordering': "['-tstamp']", 'object_name': 'TaskState'}, + 'args': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'eta': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'expires': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'kwargs': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'db_index': 'True'}), + 'result': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'retries': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'runtime': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'state': ('django.db.models.fields.CharField', [], {'max_length': '64', 'db_index': 'True'}), + 'task_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '36'}), + 'traceback': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'tstamp': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'worker': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djcelery.WorkerState']", 'null': 'True'}) + }, + 'djcelery.workerstate': { + 'Meta': {'ordering': "['-last_heartbeat']", 'object_name': 'WorkerState'}, + 'hostname': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_heartbeat': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}) + }, + 'djeuscan.categoryassociation': { + 'Meta': {'unique_together': "(['user', 'category'],)", 'object_name': 'CategoryAssociation'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.categorylog': { + 'Meta': {'object_name': 'CategoryLog', '_ormbases': ['djeuscan.Log']}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.euscanresult': { + 'Meta': {'object_name': 'EuscanResult'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'ebuild': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'result': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'scan_time': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}) + }, + 'djeuscan.herd': { + 'Meta': {'object_name': 'Herd'}, + 'email': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'herd': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False'}) + }, + 'djeuscan.herdassociation': { + 'Meta': {'unique_together': "(['user', 'herd'],)", 'object_name': 'HerdAssociation'}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.herdlog': { + 'Meta': {'object_name': 'HerdLog', '_ormbases': ['djeuscan.Log']}, + 'herd': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Herd']"}), + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'djeuscan.log': { + 'Meta': {'object_name': 'Log'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'n_packages_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_outdated': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packages_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_gentoo': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions_upstream': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'djeuscan.maintainer': { + 'Meta': {'object_name': 'Maintainer'}, + 'email': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.maintainerassociation': { + 'Meta': {'unique_together': "(['user', 'maintainer'],)", 'object_name': 'MaintainerAssociation'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.maintainerlog': { + 'Meta': {'object_name': 'MaintainerLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Maintainer']"}) + }, + 'djeuscan.overlayassociation': { + 'Meta': {'unique_together': "(['user', 'overlay'],)", 'object_name': 'OverlayAssociation'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.package': { + 'Meta': {'unique_together': "(['category', 'name'],)", 'object_name': 'Package'}, + 'category': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'herds': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Herd']", 'symmetrical': 'False', 'blank': 'True'}), + 'homepage': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_version_gentoo': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_gentoo'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_overlay': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_overlay'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'last_version_upstream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'last_version_upstream'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['djeuscan.Version']"}), + 'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['djeuscan.Maintainer']", 'symmetrical': 'False', 'blank': 'True'}), + 'n_overlay': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_packaged': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'n_versions': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'djeuscan.packageassociation': { + 'Meta': {'unique_together': "(['user', 'package'],)", 'object_name': 'PackageAssociation'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'djeuscan.problemreport': { + 'Meta': {'object_name': 'ProblemReport'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'version': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Version']", 'null': 'True', 'blank': 'True'}) + }, + 'djeuscan.refreshpackagequery': { + 'Meta': {'object_name': 'RefreshPackageQuery'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'priority': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'users': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'symmetrical': 'False'}) + }, + 'djeuscan.version': { + 'Meta': {'unique_together': "(['package', 'slot', 'revision', 'version', 'overlay'],)", 'object_name': 'Version'}, + 'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'confidence': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'ebuild_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'handler': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'metadata_path': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'db_index': 'True', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'urls': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.versionlog': { + 'Meta': {'object_name': 'VersionLog'}, + 'action': ('django.db.models.fields.IntegerField', [], {}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'overlay': ('django.db.models.fields.CharField', [], {'default': "'gentoo'", 'max_length': '128', 'blank': 'True'}), + 'package': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djeuscan.Package']"}), + 'packaged': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'slot': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'vtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'blank': 'True'}) + }, + 'djeuscan.worldlog': { + 'Meta': {'object_name': 'WorldLog', '_ormbases': ['djeuscan.Log']}, + 'log_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['djeuscan.Log']", 'unique': 'True', 'primary_key': 'True'}) + } + } + + complete_apps = ['djcelery', 'djeuscan'] + symmetrical = True From 67654ee0b4da40741e2eb0a6d9c00d823a2031c5 Mon Sep 17 00:00:00 2001 From: volpino Date: Tue, 24 Jul 2012 08:10:56 +0200 Subject: [PATCH 07/14] euscan: Progressbar on stderr Signed-off-by: volpino --- bin/euscan | 3 ++- euscanwww/djeuscan/processing/scan/scan_portage.py | 1 + euscanwww/djeuscan/tests/__init__.py | 1 - pym/euscan/out.py | 2 ++ 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/euscan b/bin/euscan index eb637d1..47bb7b4 100755 --- a/bin/euscan +++ b/bin/euscan @@ -35,6 +35,7 @@ from euscan import CONFIG, output # Globals isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() +isatty_stderr = os.environ.get('TERM') != 'dumb' and sys.stderr.isatty() def exit_helper(status): @@ -183,7 +184,7 @@ def parse_args(): CONFIG['nocolor'] = True pp.output.nocolor() elif o in ("-p", "--progress"): - CONFIG['progress'] = isatty + CONFIG['progress'] = isatty_stderr elif o in ("--mirror"): CONFIG['mirror'] = True elif o in ("-i", "--ignore-pre-release"): diff --git a/euscanwww/djeuscan/processing/scan/scan_portage.py b/euscanwww/djeuscan/processing/scan/scan_portage.py index 79fa62a..f4ca15d 100644 --- a/euscanwww/djeuscan/processing/scan/scan_portage.py +++ b/euscanwww/djeuscan/processing/scan/scan_portage.py @@ -13,6 +13,7 @@ from euscan.version import get_version_type from djeuscan.processing import FakeLogger from djeuscan.models import Package, Version, VersionLog + class ScanPortage(object): def __init__(self, logger=None, no_log=False, purge_packages=False, purge_versions=False): diff --git a/euscanwww/djeuscan/tests/__init__.py b/euscanwww/djeuscan/tests/__init__.py index 5e37ff2..78a1082 100644 --- a/euscanwww/djeuscan/tests/__init__.py +++ b/euscanwww/djeuscan/tests/__init__.py @@ -3,7 +3,6 @@ from urllib import urlencode from django.test import TestCase from django.test.client import Client from django.core.urlresolvers import reverse -from django.contrib.auth.models import User from djeuscan.tests.euscan_factory import UserFactory diff --git a/pym/euscan/out.py b/pym/euscan/out.py index c0ed61b..8a505fc 100644 --- a/pym/euscan/out.py +++ b/pym/euscan/out.py @@ -1,3 +1,4 @@ +import sys from io import StringIO from collections import defaultdict import json @@ -37,6 +38,7 @@ class ProgressHandler(object): def progress_bar(): on_progress = None progress_bar = TermProgressBar(title="euscan") + progress_bar.file = sys.stderr progress_handler = ProgressHandler(progress_bar) on_progress = progress_handler.on_progress From 1cf5d0726a09bfba2b3a728f65cd2f8a8ec74dad Mon Sep 17 00:00:00 2001 From: volpino Date: Tue, 24 Jul 2012 08:56:00 +0200 Subject: [PATCH 08/14] euscan: handlers now use a Package object instead of cpv Signed-off-by: volpino --- pym/euscan/handlers/__init__.py | 16 ++++++++-------- pym/euscan/handlers/cpan.py | 8 ++++---- pym/euscan/handlers/generic.py | 10 +++++----- pym/euscan/handlers/github.py | 10 +++++----- pym/euscan/handlers/kde.py | 14 +++++++------- pym/euscan/handlers/php.py | 14 +++++++------- pym/euscan/handlers/pypi.py | 10 +++++----- pym/euscan/handlers/rubygem.py | 12 ++++++------ pym/euscan/scan.py | 10 +++++----- 9 files changed, 52 insertions(+), 52 deletions(-) diff --git a/pym/euscan/handlers/__init__.py b/pym/euscan/handlers/__init__.py index bcbd680..495ef7c 100644 --- a/pym/euscan/handlers/__init__.py +++ b/pym/euscan/handlers/__init__.py @@ -14,22 +14,22 @@ handlers = sorted( ) -def find_best_handler(cpv, url): +def find_best_handler(pkg, url): for handler in handlers: - if handler.can_handle(cpv, url): + if handler.can_handle(pkg, url): return handler return None -def scan(cpv, url): - handler = find_best_handler(cpv, url) +def scan(pkg, url): + handler = find_best_handler(pkg, url) if handler: - return handler.scan(cpv, url) + return handler.scan(pkg, url) return [] -def brute_force(cpv, url): - handler = find_best_handler(cpv, url) +def brute_force(pkg, url): + handler = find_best_handler(pkg, url) if handler: - return handler.brute_force(cpv, url) + return handler.brute_force(pkg, url) return [] diff --git a/pym/euscan/handlers/cpan.py b/pym/euscan/handlers/cpan.py index 7115c20..091c64c 100644 --- a/pym/euscan/handlers/cpan.py +++ b/pym/euscan/handlers/cpan.py @@ -12,7 +12,7 @@ PRIORITY = 100 _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*") -def can_handle(cpv, url): +def can_handle(pkg, url): return url.startswith('mirror://cpan/') @@ -79,8 +79,8 @@ def cpan_vercmp(cp, a, b): return 1 -def scan(cpv, url): - cp, ver, rev = portage.pkgsplit(cpv) +def scan(pkg, url): + cp, ver, rev = portage.pkgsplit(pkg.cpv) pkg = guess_package(cp, url) orig_url = url @@ -133,5 +133,5 @@ def scan(cpv, url): return ret -def brute_force(cpv, url): +def brute_force(pkg, url): return [] diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/generic.py index 5c6d467..a9a3048 100644 --- a/pym/euscan/handlers/generic.py +++ b/pym/euscan/handlers/generic.py @@ -104,7 +104,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): return versions -def scan(cpv, url): +def scan(pkg, url): for bu in SCANDIR_BLACKLIST_URLS: if re.match(bu, url): output.einfo("%s is blacklisted by rule %s" % (url, bu)) @@ -114,7 +114,7 @@ def scan(cpv, url): if not resolved_url: return [] - cp, ver, rev = portage.pkgsplit(cpv) + cp, ver, rev = portage.pkgsplit(pkg.cpv) # 'Hack' for _beta/_rc versions where _ is used instead of - if ver not in resolved_url: @@ -141,8 +141,8 @@ def scan(cpv, url): return ret -def brute_force(cpv, url): - cp, ver, rev = portage.pkgsplit(cpv) +def brute_force(pkg, url): + cp, ver, rev = portage.pkgsplit(pkg.cpv) url = helpers.parse_mirror(url) if not url: @@ -226,5 +226,5 @@ def brute_force(cpv, url): return result -def can_handle(cpv, url): +def can_handle(pkg, url): return True diff --git a/pym/euscan/handlers/github.py b/pym/euscan/handlers/github.py index ac5baca..76c50c9 100644 --- a/pym/euscan/handlers/github.py +++ b/pym/euscan/handlers/github.py @@ -11,7 +11,7 @@ CONFIDENCE = 100.0 PRIORITY = 100 -def can_handle(cpv, url): +def can_handle(pkg, url): return url.startswith('mirror://github/') @@ -22,13 +22,13 @@ def guess_package(cp, url): return (match.group(1), match.group(2), match.group(3)) -def scan(cpv, url): +def scan(pkg, url): 'http://developer.github.com/v3/repos/downloads/' - user, project, filename = guess_package(cpv, url) + user, project, filename = guess_package(pkg.cpv, url) # find out where version is expected to be found - cp, ver, rev = portage.pkgsplit(cpv) + cp, ver, rev = portage.pkgsplit(pkg.cpv) if ver not in filename: return @@ -54,5 +54,5 @@ def scan(cpv, url): yield (dl['html_url'], pv, HANDLER_NAME, CONFIDENCE) -def brute_force(cpv, url): +def brute_force(pkg, url): return [] diff --git a/pym/euscan/handlers/kde.py b/pym/euscan/handlers/kde.py index 406b079..1dcead6 100644 --- a/pym/euscan/handlers/kde.py +++ b/pym/euscan/handlers/kde.py @@ -5,7 +5,7 @@ PRIORITY = 100 HANDLER_NAME = "kde" -def can_handle(cpv, url): +def can_handle(pkg, url): if url.startswith('mirror://kde/'): return True return False @@ -22,21 +22,21 @@ def clean_results(results): return ret -def scan(cpv, url): - results = generic.scan(cpv, url) +def scan(pkg, url): + results = generic.scan(pkg.cpv, url) if url.startswith('mirror://kde/unstable/'): url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/') - results += generic.scan(cpv, url) + results += generic.scan(pkg.cpv, url) return clean_results(results) -def brute_force(cpv, url): - results = generic.brute_force(cpv, url) +def brute_force(pkg, url): + results = generic.brute_force(pkg.cpv, url) if url.startswith('mirror://kde/unstable/'): url = url.replace('mirror://kde/unstable/', 'mirror://kde/stable/') - results += generic.brute_force(cpv, url) + results += generic.brute_force(pkg.cpv, url) return clean_results(results) diff --git a/pym/euscan/handlers/php.py b/pym/euscan/handlers/php.py index 158280c..36bf3d7 100644 --- a/pym/euscan/handlers/php.py +++ b/pym/euscan/handlers/php.py @@ -10,7 +10,7 @@ CONFIDENCE = 100.0 PRIORITY = 100 -def can_handle(cpv, url): +def can_handle(pkg, url): if url.startswith('http://pear.php.net/get/'): return True if url.startswith('http://pecl.php.net/get/'): @@ -30,12 +30,12 @@ def guess_package_and_channel(cp, url): return pkg, host -def scan(cpv, url): - cp, ver, rev = portage.pkgsplit(cpv) - pkg, channel = guess_package_and_channel(cp, url) +def scan(pkg, url): + cp, ver, rev = portage.pkgsplit(pkg.cpv) + package, channel = guess_package_and_channel(cp, url) orig_url = url - url = 'http://%s/rest/r/%s/allreleases.xml' % (channel, pkg.lower()) + url = 'http://%s/rest/r/%s/allreleases.xml' % (channel, package.lower()) output.einfo("Using: " + url) @@ -62,7 +62,7 @@ def scan(cpv, url): if helpers.version_filtered(cp, ver, pv): continue - url = 'http://%s/get/%s-%s.tgz' % (channel, pkg, up_pv) + url = 'http://%s/get/%s-%s.tgz' % (channel, package, up_pv) if url == orig_url: continue @@ -72,5 +72,5 @@ def scan(cpv, url): return ret -def brute_force(cpv, url): +def brute_force(pkg, url): return [] diff --git a/pym/euscan/handlers/pypi.py b/pym/euscan/handlers/pypi.py index 1dcad99..74aaeb3 100644 --- a/pym/euscan/handlers/pypi.py +++ b/pym/euscan/handlers/pypi.py @@ -10,7 +10,7 @@ CONFIDENCE = 100.0 PRIORITY = 100 -def can_handle(cpv, url): +def can_handle(pkg, url): return url.startswith('mirror://pypi/') @@ -24,10 +24,10 @@ def guess_package(cp, url): return pkg -def scan(cpv, url): +def scan(pkg, url): 'http://wiki.python.org/moin/PyPiXmlRpc' - package = guess_package(cpv, url) + package = guess_package(pkg.cpv, url) output.einfo("Using PyPi XMLRPC: " + package) @@ -39,7 +39,7 @@ def scan(cpv, url): versions.reverse() - cp, ver, rev = portage.pkgsplit(cpv) + cp, ver, rev = portage.pkgsplit(pkg.cpv) ret = [] @@ -54,5 +54,5 @@ def scan(cpv, url): return ret -def brute_force(cpv, url): +def brute_force(pkg, url): return [] diff --git a/pym/euscan/handlers/rubygem.py b/pym/euscan/handlers/rubygem.py index c0f7ade..529e6d4 100644 --- a/pym/euscan/handlers/rubygem.py +++ b/pym/euscan/handlers/rubygem.py @@ -10,7 +10,7 @@ CONFIDENCE = 100.0 PRIORITY = 100 -def can_handle(cpv, url): +def can_handle(pkg, url): return url.startswith('mirror://rubygems/') @@ -29,13 +29,13 @@ def guess_gem(cpv, url): return pkg -def scan(cpv, url): +def scan(pkg, url): 'http://guides.rubygems.org/rubygems-org-api/#gemversion' - gem = guess_gem(cpv, url) + gem = guess_gem(pkg.cpv, url) if not gem: output.eerror("Can't guess gem name using %s and %s" % \ - (cpv, url)) + (pkg.cpv, url)) return [] url = 'http://rubygems.org/api/v1/versions/%s.json' % gem @@ -58,7 +58,7 @@ def scan(cpv, url): if not versions: return [] - cp, ver, rev = portage.pkgsplit(cpv) + cp, ver, rev = portage.pkgsplit(pkg.cpv) ret = [] @@ -73,5 +73,5 @@ def scan(cpv, url): return ret -def brute_force(cpv, url): +def brute_force(pkg, url): return [] diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py index d63488b..114c81e 100644 --- a/pym/euscan/scan.py +++ b/pym/euscan/scan.py @@ -44,7 +44,7 @@ def filter_versions(cp, versions): ] -def scan_upstream_urls(cpv, urls, on_progress): +def scan_upstream_urls(pkg, urls, on_progress): versions = [] if on_progress: @@ -73,7 +73,7 @@ def scan_upstream_urls(cpv, urls, on_progress): # Try normal scan if CONFIG["scan-dir"]: try: - versions.extend(handlers.scan(cpv, url)) + versions.extend(handlers.scan(pkg, url)) except Exception as e: output.ewarn("Handler failed: [%s] %s" % (e.__class__.__name__, e.message)) @@ -83,12 +83,12 @@ def scan_upstream_urls(cpv, urls, on_progress): # Brute Force if CONFIG["brute-force"] > 0: - versions.extend(handlers.brute_force(cpv, url)) + versions.extend(handlers.brute_force(pkg, url)) if versions and CONFIG['oneshot']: break - cp, ver, rev = portage.pkgsplit(cpv) + cp, ver, rev = portage.pkgsplit(pkg.cpv) result = filter_versions(cp, versions) @@ -216,7 +216,7 @@ def scan_upstream(query, on_progress=None): scan_time = (datetime.now() - start_time).total_seconds() output.metadata("scan_time", scan_time, show=False) - result = scan_upstream_urls(pkg.cpv, urls, on_progress) + result = scan_upstream_urls(pkg, urls, on_progress) if on_progress: on_progress(increment=10) From 795dcb141481982060253737dd2c03f06048c2a8 Mon Sep 17 00:00:00 2001 From: volpino Date: Tue, 24 Jul 2012 09:03:14 +0200 Subject: [PATCH 09/14] euscanwww: == None -> is None Signed-off-by: volpino --- euscanwww/djeuscan/processing/scan/scan_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/euscanwww/djeuscan/processing/scan/scan_metadata.py b/euscanwww/djeuscan/processing/scan/scan_metadata.py index ee028d4..54dd9c4 100644 --- a/euscanwww/djeuscan/processing/scan/scan_metadata.py +++ b/euscanwww/djeuscan/processing/scan/scan_metadata.py @@ -170,7 +170,7 @@ def scan_metadata(packages=None, category=None, logger=None, populate=False): if category: packages = Package.objects.filter(category=category) - elif packages == None: + elif packages is None: packages = Package.objects.all() if populate: From e2890c636419b3209e3b2e6405c45e541568f870 Mon Sep 17 00:00:00 2001 From: volpino Date: Tue, 24 Jul 2012 15:02:36 +0200 Subject: [PATCH 10/14] euscan: Added watch handler, fixed generic one Signed-off-by: volpino --- bin/euscan_patch_metadata | 91 +++++++++++++-------- euscanwww/djeuscan/views.py | 2 +- pym/euscan/handlers/cpan.py | 2 +- pym/euscan/handlers/generic.py | 14 ++-- pym/euscan/handlers/github.py | 2 +- pym/euscan/handlers/kde.py | 4 +- pym/euscan/handlers/php.py | 2 +- pym/euscan/handlers/pypi.py | 2 +- pym/euscan/handlers/rubygem.py | 2 +- pym/euscan/handlers/watch.py | 139 +++++++++++++++++++++++++++++++++ pym/euscan/helpers.py | 2 + 11 files changed, 213 insertions(+), 49 deletions(-) create mode 100644 pym/euscan/handlers/watch.py diff --git a/bin/euscan_patch_metadata b/bin/euscan_patch_metadata index b5649ed..0de4239 100755 --- a/bin/euscan_patch_metadata +++ b/bin/euscan_patch_metadata @@ -11,6 +11,7 @@ import logging import shutil import subprocess +from portage.exception import AmbiguousPackageName from gentoolkit.query import Query from BeautifulSoup import BeautifulSoup, SoupStrainer @@ -133,40 +134,60 @@ def get_deb_url(name): def patch_metadata(metadata_path, watch_data, diff=False): - watch_data = "\n".join([line for line in watch_data.split("\n") - if not line.startswith("#")]) # comments - watch_data = watch_data.replace("\\\n", "") # remove backslashes - watch_data = " ".join(watch_data.split()) # remove extra spaces and \n - - result = re.match( - r'(version=\d+?) (?:opts=(?:"([^"]+?)"|([^\s]+?)) )?(.*)', watch_data - ) - - version, attrs_quote, attrs, url = result.groups() - attrs = attrs_quote or attrs - - if attrs: - attrs = [x.replace('=', '="') + '"' for x in attrs.split(",")] - attrs = " ".join(attrs) + logger.info(" Patching metadata file") with open(metadata_path) as fp: original = fp.read() rindent, indent = guess_indent_values(original) - data = original - logger.info(" Patching metadata file") + # clean watch_data + watch_data = "\n".join([line for line in watch_data.split("\n") + if not line.startswith("#")]) # comments - if attrs: - watch_tag = '%s%s' % (indent, version, attrs, url) - else: - watch_tag = '%s%s' % (indent, version, url) + watch_data = watch_data.replace("\\\n", "") # remove backslashes + + watch_tags = [] + + for watch_line in watch_data.split("\n"): # there can be multiple lines + watch_line = " ".join(watch_line.split()) # remove extra spaces and \n + + version_parse = re.match("version=(\d+?)", watch_line) + if version_parse: + version = version_parse.group(1) + continue + + if not watch_line: # skip empty lines + continue + + # parse watch_line + result = re.match( + r'(?:opts=(?:"([^"]+?)"|([^\s]+?)) )?(.*)', + watch_line + ) + + attrs_quote, attrs, url = result.groups() + attrs = attrs_quote or attrs + + if attrs: + attrs = [x.replace('=', '="') + '"' for x in attrs.split(",")] + attrs = " ".join(attrs) + + if attrs: + watch_tag = '%s%s' % \ + (indent, version, attrs, url) + else: + watch_tag = '%s%s' % \ + (indent, version, url) + watch_tags.append(watch_tag) + + watch_tags = "\n".join(watch_tags) if '' in data: - data = data.replace('', '\n%s' % watch_tag, 1) + data = data.replace('', '\n%s' % watch_tags, 1) else: rep = '%s\n%s\n%s\n' % \ - (rindent, watch_tag, rindent) + (rindent, watch_tags, rindent) data = data.replace('', rep, 1) if not diff: @@ -183,14 +204,18 @@ def patch_metadata(metadata_path, watch_data, diff=False): def process_package(query, diff=False): - matches = Query(query).smart_find( - in_installed=True, - in_porttree=True, - in_overlay=True, - include_masked=True, - show_progress=False, - no_matches_fatal=False, - ) + try: + matches = Query(query).smart_find( + in_installed=True, + in_porttree=True, + in_overlay=True, + include_masked=True, + show_progress=False, + no_matches_fatal=False, + ) + except AmbiguousPackageName: + logger.error(" Ambiguous package name") + return None if len(matches) == 0: logger.error(" Package not found") @@ -224,7 +249,9 @@ def main(): for package in packages: logger.info("Processing %s..." % package) - print process_package(package, opts.diff) + result = process_package(package, opts.diff) + if result: + print result if __name__ == "__main__": main() diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py index af7d7be..3d96cb0 100644 --- a/euscanwww/djeuscan/views.py +++ b/euscanwww/djeuscan/views.py @@ -250,7 +250,7 @@ def package(request, category, package): 'upstream': upstream, 'log': log, 'vlog': vlog, - 'msg' : msg, + 'msg': msg, 'last_scan': last_scan, 'favourited': favourited, 'refreshed': refreshed, diff --git a/pym/euscan/handlers/cpan.py b/pym/euscan/handlers/cpan.py index 091c64c..a54641f 100644 --- a/pym/euscan/handlers/cpan.py +++ b/pym/euscan/handlers/cpan.py @@ -7,7 +7,7 @@ from euscan import helpers, output HANDLER_NAME = "cpan" CONFIDENCE = 100.0 -PRIORITY = 100 +PRIORITY = 90 _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*") diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/generic.py index a9a3048..831dfce 100644 --- a/pym/euscan/handlers/generic.py +++ b/pym/euscan/handlers/generic.py @@ -1,3 +1,4 @@ +from urlparse import urljoin import urllib2 import re import StringIO @@ -34,8 +35,7 @@ def scan_html(data, url, pattern): match = re.match(pattern, href, re.I) if match: - results.append((match.group(1), match.group(0))) - + results.append((".".join(match.groups()), match.group(0))) return results @@ -47,7 +47,7 @@ def scan_ftp(data, url, pattern): line = line.replace("\n", "").replace("\r", "") match = re.search(pattern, line, re.I) if match: - results.append((match.group(1), match.group(0))) + results.append((".".join(match.groups()), match.group(0))) return results @@ -77,7 +77,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): results = [] - if re.search("<\s*a\s+[^>]*href", data): + if re.search("<\s*a\s+[^>]*href", data, re.I): results.extend(scan_html(data, url, pattern)) elif url.startswith('ftp://'): results.extend(scan_ftp(data, url, pattern)) @@ -88,11 +88,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): pv = helpers.gentoo_mangle_version(up_pv) if helpers.version_filtered(cp, ver, pv): continue - - if not url.endswith('/') and not path.startswith('/'): - path = url + '/' + path - else: - path = url + path + path = urljoin(url, path) if not steps and path not in orig_url: versions.append((path, pv, HANDLER_NAME, CONFIDENCE)) diff --git a/pym/euscan/handlers/github.py b/pym/euscan/handlers/github.py index 76c50c9..9bb5596 100644 --- a/pym/euscan/handlers/github.py +++ b/pym/euscan/handlers/github.py @@ -8,7 +8,7 @@ from euscan import helpers, output HANDLER_NAME = "github" CONFIDENCE = 100.0 -PRIORITY = 100 +PRIORITY = 90 def can_handle(pkg, url): diff --git a/pym/euscan/handlers/kde.py b/pym/euscan/handlers/kde.py index 1dcead6..21722bb 100644 --- a/pym/euscan/handlers/kde.py +++ b/pym/euscan/handlers/kde.py @@ -1,6 +1,6 @@ from euscan.handlers import generic -PRIORITY = 100 +PRIORITY = 90 HANDLER_NAME = "kde" @@ -14,7 +14,7 @@ def can_handle(pkg, url): def clean_results(results): ret = [] - for path, version, confidence in results: + for path, version, _, confidence in results: if version == '5SUMS': continue ret.append((path, version, HANDLER_NAME, confidence)) diff --git a/pym/euscan/handlers/php.py b/pym/euscan/handlers/php.py index 36bf3d7..6b74ff6 100644 --- a/pym/euscan/handlers/php.py +++ b/pym/euscan/handlers/php.py @@ -7,7 +7,7 @@ from euscan import helpers, output HANDLER_NAME = "php" CONFIDENCE = 100.0 -PRIORITY = 100 +PRIORITY = 90 def can_handle(pkg, url): diff --git a/pym/euscan/handlers/pypi.py b/pym/euscan/handlers/pypi.py index 74aaeb3..9cd1620 100644 --- a/pym/euscan/handlers/pypi.py +++ b/pym/euscan/handlers/pypi.py @@ -7,7 +7,7 @@ from euscan import helpers, output HANDLER_NAME = "pypi" CONFIDENCE = 100.0 -PRIORITY = 100 +PRIORITY = 90 def can_handle(pkg, url): diff --git a/pym/euscan/handlers/rubygem.py b/pym/euscan/handlers/rubygem.py index 529e6d4..39e2334 100644 --- a/pym/euscan/handlers/rubygem.py +++ b/pym/euscan/handlers/rubygem.py @@ -7,7 +7,7 @@ from euscan import helpers, output HANDLER_NAME = "rubygem" CONFIDENCE = 100.0 -PRIORITY = 100 +PRIORITY = 90 def can_handle(pkg, url): diff --git a/pym/euscan/handlers/watch.py b/pym/euscan/handlers/watch.py new file mode 100644 index 0000000..d172072 --- /dev/null +++ b/pym/euscan/handlers/watch.py @@ -0,0 +1,139 @@ +import re +import urllib2 + +import portage + +from euscan.handlers import generic +from euscan import output, helpers + +PRIORITY = 100 + +HANDLER_NAME = "watch" +CONFIDENCE = 100.0 + + +is_pattern = r"\([^\/]+\)" + + +def can_handle(pkg, url): + try: + return pkg.metadata._xml_tree.find("upstream").find("watch") \ + is not None + except AttributeError: + return False + + +def parse_mangles(mangles, string): + for mangle in mangles: + # convert regex from perl format to python format + m = re.match(r"s/(.*[^\\])/(.*)/", mangle) + pattern, repl = m.groups() + repl = re.sub(r"\$(\d+)", r"\\\1", repl) + string = re.sub(pattern, repl, string) + return string + + +def clean_results(results, versionmangle, urlmangle): + ret = [] + + for path, version, _, _ in results: + version = parse_mangles(versionmangle, version) + path = parse_mangles(urlmangle, path) + ret.append((path, version, HANDLER_NAME, CONFIDENCE)) + + return ret + + +def parse_watch(pkg): + for watch_tag in pkg.metadata._xml_tree.find("upstream").findall("watch"): + try: + base, file_pattern = watch_tag.text.split(" ")[:2] + except ValueError: + base, file_pattern = watch_tag.text, None + + # the file pattern can be in the base url + pattern_regex = r"/([^/]*\([^/]*\)[^/]*)$" + match = re.search(pattern_regex, base) + if match: + file_pattern = match.group(1) + base = base.replace(file_pattern, "") + + # handle sf.net specially + base = base.replace( + "http://sf.net/", "http://qa.debian.org/watch/sf.php/" + ) + + vmangle = watch_tag.attrib.get("uversionmangle", None) or \ + watch_tag.attrib.get("versionmangle", None) + versionmangle = vmangle.split(";") if vmangle else [] + + umangle = watch_tag.attrib.get("downloadurlmangle", None) + urlmangle = umangle.split(";") if umangle else [] + + yield (base, file_pattern, versionmangle, urlmangle) + + +def handle_directory_patterns(base, file_pattern): + """ + Directory pattern matching + e.g.: base: ftp://ftp.nessus.org/pub/nessus/nessus-([\d\.]+)/src/ + file_pattern: nessus-core-([\d\.]+)\.tar\.gz + """ + splitted = base.split("/") + i = 0 + basedir = [] + for elem in splitted: + if re.search(is_pattern, elem): + break + basedir.append(elem) + i += 1 + basedir = "/".join(basedir) + directory_pattern = splitted[i] + final = "/".join(splitted[i + 1:]) + + try: + fp = helpers.urlopen(basedir) + except urllib2.URLError: + return [] + except IOError: + return [] + + if not fp: + return [] + + data = fp.read() + + if basedir.startswith("ftp://"): + scan_data = generic.scan_ftp(data, basedir, directory_pattern) + else: + scan_data = generic.scan_html(data, basedir, directory_pattern) + + return [("/".join((basedir, path, final)), file_pattern) + for _, path in scan_data] + + +def scan(pkg, url): + output.einfo("Using watch data") + + cp, ver, rev = portage.pkgsplit(pkg.cpv) + + results = [] + for base, file_pattern, versionmangle, urlmangle in parse_watch(pkg): + if not re.search(is_pattern, base): + steps = [(base, file_pattern)] + res = generic.scan_directory_recursive( + cp, ver, rev, "", steps, url + ) + else: + res = [] + for step in handle_directory_patterns(base, file_pattern): + res += generic.scan_directory_recursive( + cp, ver, rev, "", [step], url + ) + + results += clean_results(res, versionmangle, urlmangle) + return results + + +def brute_force(pkg, url): + return [] diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py index 1e385cd..6582393 100644 --- a/pym/euscan/helpers.py +++ b/pym/euscan/helpers.py @@ -33,6 +33,7 @@ VERSION_CMP_PACKAGE_QUIRKS = { _v_end = '((-|_)(pre|p|beta|b|alpha|a|rc|r)\d*)' _v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))' + # Stolen from g-pypi def gentoo_mangle_version(up_pv): """Convert PV to MY_PV if needed @@ -537,6 +538,7 @@ def generate_scan_paths(url): return steps + def parse_mirror(uri): from random import shuffle From 9f164ed22c55a17e641ff2e9107116fa67af3035 Mon Sep 17 00:00:00 2001 From: volpino Date: Wed, 25 Jul 2012 10:19:27 +0200 Subject: [PATCH 11/14] euscanwww: Removing useless stuff while creating watch tag * Removing useless options or actions * Fixing handling of substitution regex Signed-off-by: volpino --- bin/euscan_patch_metadata | 24 +++++++++++++++++------- pym/euscan/handlers/watch.py | 4 ++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/bin/euscan_patch_metadata b/bin/euscan_patch_metadata index 0de4239..642b639 100755 --- a/bin/euscan_patch_metadata +++ b/bin/euscan_patch_metadata @@ -166,16 +166,26 @@ def patch_metadata(metadata_path, watch_data, diff=False): watch_line ) - attrs_quote, attrs, url = result.groups() - attrs = attrs_quote or attrs + opts_quote, opts, url = result.groups() + opts = opts_quote or opts - if attrs: - attrs = [x.replace('=', '="') + '"' for x in attrs.split(",")] - attrs = " ".join(attrs) + if opts: + # clean opts, skip useless ones + valid = ("uversionmangle", "versionmangle", "downloadurlmangle") + cleaned_opts = [] + for opt in opts.split(","): + opt_name, opt_value = opt.split("=") + if opt_name in valid: + cleaned_opts.append('%s="%s"' % (opt_name, opt_value)) + opts = " ".join(cleaned_opts) - if attrs: + # clean url from useless stuff. Just keep [] + url_search = re.search(r"^([^\s]+)(?: ([^\s]*\([^\s]+\)[^\s]*))?", url) + url = " ".join([x for x in url_search.groups() if x is not None]) + + if opts: watch_tag = '%s%s' % \ - (indent, version, attrs, url) + (indent, version, opts, url) else: watch_tag = '%s%s' % \ (indent, version, url) diff --git a/pym/euscan/handlers/watch.py b/pym/euscan/handlers/watch.py index d172072..a129281 100644 --- a/pym/euscan/handlers/watch.py +++ b/pym/euscan/handlers/watch.py @@ -26,7 +26,11 @@ def can_handle(pkg, url): def parse_mangles(mangles, string): for mangle in mangles: # convert regex from perl format to python format + # there are some regex in this format: s/pattern/replacement/ m = re.match(r"s/(.*[^\\])/(.*)/", mangle) + if not m: + # or in this format s|pattern|replacement| + m = re.match(r"s\|(.*[^\\])\|(.*)\|", mangle) pattern, repl = m.groups() repl = re.sub(r"\$(\d+)", r"\\\1", repl) string = re.sub(pattern, repl, string) From 112313ab7f9b5e7230cbf4c867c5b8a89604ac44 Mon Sep 17 00:00:00 2001 From: volpino Date: Wed, 25 Jul 2012 10:44:15 +0200 Subject: [PATCH 12/14] euscan: generic handler fixes * urljoin was broken with dirs ("http://site.com/lol" joined with "wat" produces http://site.com/wat) * fixed _v regex, don't match what is not needed Signed-off-by: volpino --- pym/euscan/handlers/generic.py | 13 ++++++++++--- pym/euscan/helpers.py | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/generic.py index 831dfce..12c4a99 100644 --- a/pym/euscan/handlers/generic.py +++ b/pym/euscan/handlers/generic.py @@ -35,7 +35,10 @@ def scan_html(data, url, pattern): match = re.match(pattern, href, re.I) if match: - results.append((".".join(match.groups()), match.group(0))) + results.append( + (".".join([x for x in match.groups() if x is not None]), + match.group(0)) + ) return results @@ -47,8 +50,10 @@ def scan_ftp(data, url, pattern): line = line.replace("\n", "").replace("\r", "") match = re.search(pattern, line, re.I) if match: - results.append((".".join(match.groups()), match.group(0))) - + results.append( + (".".join([x for x in match.groups() if x is not None]), + match.group(0)) + ) return results @@ -88,6 +93,8 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): pv = helpers.gentoo_mangle_version(up_pv) if helpers.version_filtered(cp, ver, pv): continue + if not url.endswith("/"): + url = url + "/" path = urljoin(url, path) if not steps and path not in orig_url: diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py index 6582393..ec721b7 100644 --- a/pym/euscan/helpers.py +++ b/pym/euscan/helpers.py @@ -30,8 +30,8 @@ VERSION_CMP_PACKAGE_QUIRKS = { 'sys-process/htop': htop_vercmp } -_v_end = '((-|_)(pre|p|beta|b|alpha|a|rc|r)\d*)' -_v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))' +_v_end = r'(?:(?:-|_)(?:pre|p|beta|b|alpha|a|rc|r)\d*)' +_v = r'((?:\d+)(?:(?:\.\d+)*)(?:[a-zA-Z]*?)(?:' + _v_end + '*))' # Stolen from g-pypi From a12e2c42ea4f449fb3d12bb13a64c232468eb6d6 Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Wed, 25 Jul 2012 16:52:10 +0200 Subject: [PATCH 13/14] djeuscan: move problem to another page, use an icon, restrict to registered We may move it back to the package page later. Signed-off-by: Corentin Chary --- euscanwww/djeuscan/models.py | 10 ++ .../templates/euscan/_package_details.html | 122 +++++++++++++ .../djeuscan/templates/euscan/package.html | 167 +----------------- .../djeuscan/templates/euscan/problem.html | 57 ++++++ euscanwww/djeuscan/urls.py | 2 + euscanwww/djeuscan/views.py | 69 +++++--- euscanwww/htdocs/img/bug.png | Bin 0 -> 925 bytes 7 files changed, 239 insertions(+), 188 deletions(-) create mode 100644 euscanwww/djeuscan/templates/euscan/_package_details.html create mode 100644 euscanwww/djeuscan/templates/euscan/problem.html create mode 100644 euscanwww/htdocs/img/bug.png diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py index 0c38909..6626a41 100644 --- a/euscanwww/djeuscan/models.py +++ b/euscanwww/djeuscan/models.py @@ -1,3 +1,5 @@ +import json + from django.db import models from django.core.validators import RegexValidator, validate_email, URLValidator from django.core.exceptions import ValidationError @@ -215,6 +217,14 @@ class EuscanResult(models.Model): self.full_clean() super(EuscanResult, self).save(*args, **kwargs) + def messages(self): + result = json.loads(self.result) + + if result and self.package.cp() in result: + return result[self.package.cp()]['messages'] + else: + return "" + def __unicode__(self): return '[%s] %s/%s' % ( self.datetime, self.package.category, self.package.name diff --git a/euscanwww/djeuscan/templates/euscan/_package_details.html b/euscanwww/djeuscan/templates/euscan/_package_details.html new file mode 100644 index 0000000..1e04911 --- /dev/null +++ b/euscanwww/djeuscan/templates/euscan/_package_details.html @@ -0,0 +1,122 @@ +{% load djeuscan_helpers %} +{% load url from future %} + +
+ {% if package.description %} +
Description
+
{{ package.description }}
+ {% endif %} + {% if package.homepage %} +
Homepage
+
+ {% for homepage in package.homepages %} + {{ homepage }}
+ {% endfor %} +
+ {% endif %} + {% if package.herds.all %} +
Herds
+
+ {% for herd in package.herds.all %} + + {{ herd.herd }} + + <{{ herd.email }}> + {% endfor %} +
+ {% endif %} + {% if package.maintainers.all %} +
Maintainers
+
+ {% for maintainer in package.maintainers.all %} + {% if maintainer.name != maintainer.email %} + + {{ maintainer.name }} + + <{{ maintainer.email }}> + {% else %} + + {{ maintainer.email }} + + {% endif %} + {% endfor %} +
+ {% endif %} + {% if packaged %} +
Packaged Versions
+
+
    + {% for version in packaged %} +
  • + {% if version.overlay == "gentoo" %} + gentoo + {% else %} + overlays + {% endif %} + + {% if version.vtype and not version.vtype|is_stable %} + unstable + {% endif %} + + {{ version.version }}-{{ version.revision }} :{{ version.slot }} [{{ version.overlay }}] +
    • + {% endfor %} +
+
+ {% endif %} + {% if upstream %} +
Upstream versions
+
+
    + {% for version in upstream %} +
  • + upstream + + {% if version.vtype and not version.vtype|is_stable %} + unstable + {% endif %} + + {{ version.version }} - {{ version.urls }} + {% if confidence < 100 %}({{ version.confidence }}%){% endif %} +
    • + {% endfor %} +
+
+ {% endif %} + {% if vlog %} +
Version history
+
+
    + {% for version in vlog %} + {% if version.action == version.VERSION_ADDED %} +
  • + {% else %} +
  • + {% endif %} + {% if version.overlay == "gentoo" %} + gentoo + {% elif version.overlay %} + overlays + {% else %} + upstream + {% endif %} + + {% if version.vtype and not version.vtype|is_stable %} + unstable + {% endif %} + + {{ version }} - {{ version.datetime }} + +
    • + {% endfor %} +
+
+ {% endif %} + {% if log %} +
euscan log
+
+

Date: {{ log.datetime }} + 

{{ msg|ansi_to_html|safe }}
+
+ {% endif %} +
diff --git a/euscanwww/djeuscan/templates/euscan/package.html b/euscanwww/djeuscan/templates/euscan/package.html index 2357318..8c90efe 100644 --- a/euscanwww/djeuscan/templates/euscan/package.html +++ b/euscanwww/djeuscan/templates/euscan/package.html @@ -1,7 +1,6 @@ {% extends "euscan/_datatable.html" %} {% load sub %} -{% load djeuscan_helpers %} {% load url from future %} {% block meta %} @@ -26,12 +25,6 @@ {% block content %} -{% if thanks_for_reporting %} -
- Thanks! Your report has been sent to admins -
-{% endif %} -
A refresh request is in progress, please wait...
@@ -56,158 +49,18 @@ {% endif %} -
- {% if package.description %} -
Description
-
{{ package.description }}
- {% endif %} - {% if package.homepage %} -
Homepage
-
- {% for homepage in package.homepages %} - {{ homepage }}
- {% endfor %} -
- {% endif %} - {% if package.herds.all %} -
Herds
-
- {% for herd in package.herds.all %} - - {{ herd.herd }} - - <{{ herd.email }}> - {% endfor %} -
- {% endif %} - {% if package.maintainers.all %} -
Maintainers
-
- {% for maintainer in package.maintainers.all %} - {% if maintainer.name != maintainer.email %} - - {{ maintainer.name }} - - <{{ maintainer.email }}> - {% else %} - - {{ maintainer.email }} - - {% endif %} - {% endfor %} -
- {% endif %} - {% if packaged %} -
Packaged Versions
-
-
    - {% for version in packaged %} -
  • - {% if version.overlay == "gentoo" %} - gentoo - {% else %} - overlays - {% endif %} - {% if version.vtype and not version.vtype|is_stable %} - unstable - {% endif %} - - {{ version.version }}-{{ version.revision }} :{{ version.slot }} [{{ version.overlay }}] -
    • - {% endfor %} -
-
- {% endif %} - {% if upstream %} -
Upstream versions
-
-
    - {% for version in upstream %} -
  • - upstream - - {% if version.vtype and not version.vtype|is_stable %} - unstable - {% endif %} - - {{ version.version }} - {{ version.urls }} - {% if confidence < 100 %}({{ version.confidence }}%){% endif %} -
    • - {% endfor %} -
-
- {% endif %} -
Version history
-
-
    - {% for version in vlog %} - {% if version.action == version.VERSION_ADDED %} -
  • - {% else %} -
  • - {% endif %} - {% if version.overlay == "gentoo" %} - gentoo - {% elif version.overlay %} - overlays - {% else %} - upstream - {% endif %} - - {% if version.vtype and not version.vtype|is_stable %} - unstable - {% endif %} - - {{ version }} - {{ version.datetime }} - -
    • - {% endfor %} -
- - {% if log %} -
euscan log
-
-

Date: {{ log.datetime }} - 

{{ msg|ansi_to_html|safe }}
-
- {% endif %} -
+{% include "euscan/_package_details.html" %}
-

Report problems

-
-
-
- -
- {{ problem_form.version }} - {{ problem_form.version.errors.as_text }} -
-
-
- -
- {{ problem_form.subject }} - {{ problem_form.subject.errors.as_text }} -
-
-
- -
- {{ problem_form.message }} - {{ problem_form.message.errors.as_text }} -
-
- -
-
- -
-
-
-
+{% if user.is_authenticated %} +
+ +
+{% endif %} diff --git a/euscanwww/djeuscan/templates/euscan/problem.html b/euscanwww/djeuscan/templates/euscan/problem.html new file mode 100644 index 0000000..828b7ab --- /dev/null +++ b/euscanwww/djeuscan/templates/euscan/problem.html @@ -0,0 +1,57 @@ +{% extends "euscan/_datatable.html" %} + +{% load sub %} +{% load djeuscan_helpers %} +{% load url from future %} + +{% block title %} +{{ block.super }} - Report Problem: {{ package.category }}/{{ package.name }} +{% endblock %} + +{% block content %} + +{% if thanks_for_reporting %} +
+ Thanks! Your report has been sent to admins +
+{% endif %} + +

+ Report Problem: {{ package.category }}/{{ package.name }} +

+ +{% include "euscan/_package_details.html" %} +
+
+
+
+ +
+ {{ form.version }} + {{ form.version.errors.as_text }} +
+
+
+ +
+ {{ form.subject }} + {{ form.subject.errors.as_text }} +
+
+
+ +
+ {{ form.message }} + {{ form.message.errors.as_text }} +
+
+ +
+
+ +
+
+
+
+ +{% endblock %} diff --git a/euscanwww/djeuscan/urls.py b/euscanwww/djeuscan/urls.py index e4611d7..b1b1f88 100644 --- a/euscanwww/djeuscan/urls.py +++ b/euscanwww/djeuscan/urls.py @@ -22,6 +22,8 @@ package_patterns = patterns('djeuscan.views', 'unfavourite/$'), 'unfavourite_package', name="unfavourite_package"), url((r'^(?P[\w+][\w+.-]*)/(?P[\w+][\w+.-]*)/' 'refresh$'), "refresh_package", name="refresh_package"), + url(r'^(?P[\w+][\w+.-]*)/(?P[\w+][\w+.-]*)/problem$', + 'problem', name="problem"), ) categories_patterns = patterns('djeuscan.views', diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py index 3d96cb0..c9abfca 100644 --- a/euscanwww/djeuscan/views.py +++ b/euscanwww/djeuscan/views.py @@ -1,7 +1,6 @@ """ Views """ import inspect -import json from annoying.decorators import render_to, ajax_request from django.http import Http404 @@ -188,7 +187,6 @@ def overlay(request, overlay): return {'overlay': overlay, 'packages': packages, 'last_scan': last_scan, 'favourited': favourited} - @render_to('euscan/package.html') def package(request, category, package): package = get_object_or_404(Package, category=category, name=package) @@ -203,13 +201,6 @@ def package(request, category, package): log = log[0] if log else None vlog = VersionLog.objects.for_package(package, order=True) - result = json.loads(log.result) if log else None - - if result and package.cp() in result: - msg = result[package.cp()]['messages'] - else: - msg = "" - try: last_scan = EuscanResult.objects.for_package(package).latest().datetime except EuscanResult.DoesNotExist: @@ -230,42 +221,62 @@ def package(request, category, package): except RefreshPackageQuery.DoesNotExist: refreshed = False - thanks_for_reporting = False - if request.method == "POST": - problem_form = ProblemReportForm(package, request.POST) - if problem_form.is_valid(): - ProblemReport( - package=package, - version=problem_form.cleaned_data["version"], - subject=problem_form.cleaned_data["subject"], - message=problem_form.cleaned_data["message"], - ).save() - thanks_for_reporting = True - else: - problem_form = ProblemReportForm(package) - return { 'package': package, 'packaged': packaged, 'upstream': upstream, - 'log': log, + 'log': log.messages(), 'vlog': vlog, - 'msg': msg, + 'msg' : log.messages() if log else "", 'last_scan': last_scan, 'favourited': favourited, 'refreshed': refreshed, - 'problem_form': problem_form, - 'thanks_for_reporting': thanks_for_reporting } +@login_required +@render_to('euscan/problem.html') +def problem(request, category, package): + package = get_object_or_404(Package, category=category, name=package) + packaged = Version.objects.filter(package=package, packaged=True) + upstream = Version.objects.filter(package=package, packaged=False) + + log = EuscanResult.objects.filter(package=package).\ + order_by('-datetime')[:1] + log = log[0] if log else None + + thanks_for_reporting = False + + if request.method == "POST": + form = ProblemReportForm(package, request.POST) + if form.is_valid(): + ProblemReport( + package=package, + version=form.cleaned_data["version"], + subject=form.cleaned_data["subject"], + message=form.cleaned_data["message"], + ).save() + thanks_for_reporting = True + else: + form = ProblemReportForm(package) + + return { + 'form': form, + 'thanks_for_reporting': thanks_for_reporting, + 'package': package, + 'packaged': packaged, + 'upstream': upstream, + 'msg' : log.messages() if log else "", + } @render_to('euscan/world.html') def world(request): world_form = WorldForm() packages_form = PackagesForm() - return {'world_form': world_form, - 'packages_form': packages_form} + return { + 'world_form': world_form, + 'packages_form': packages_form + } @render_to('euscan/world_scan.html') diff --git a/euscanwww/htdocs/img/bug.png b/euscanwww/htdocs/img/bug.png new file mode 100644 index 0000000000000000000000000000000000000000..c33bfb9cfd4a9d6d242c38dc8fe5d192fc3f831e GIT binary patch literal 925 zcmV;O17iG%P)OpREm=>0CEf<6dKeK8bzA_NsB zJrK%*UWgS$StNn>A+R8%LLn$9=tB_}o&F>BMO<}*4jj(8_n!IPbIzP$7!i%A5Y>*V z5H8(WTBsv|oQO#)&BrO=CL#78YMwJcBuybxSN)1jYeb#8` zTjyg+78cU7!ZL z9uEWp^dKB=9yj=~q8xY>4AL=VYyz}ar=zC`y$36y1?pZ~Ru=Uo5_Hbt*gh^WgqteK zN3MiIbQi}@Q1UC)Lf-&(1wC(r_F&yvTugCEsv9p@GMmlyNw?=?5aqT2{19R7usvEW zb-}*~KZ2}N;Ygp=YQ_CbJ0=(-HzO1Z?O7&>4HGeL31ruZu)VO`06qhM3H$=QxJ?FS zGsdhHU|~#fG#X9j=H?zkFu#=JpCpl8J;K(*t^{}={B@|LR?rV+>TozPKo)0Ouqya~ z=7+=Kql%)K|9&*(OxwPVaf5XTU5g&CVBGf#rfpNtpq$ZYoZ|QUkH+KiNII|@v~c+E zNL^27tQ3EZb1{wPq=&I-Bg6<^uXo;X@PE Date: Wed, 25 Jul 2012 16:52:49 +0200 Subject: [PATCH 14/14] djeuscan: tweak admin.py Signed-off-by: Corentin Chary --- euscanwww/djeuscan/admin.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/euscanwww/djeuscan/admin.py b/euscanwww/djeuscan/admin.py index a64007d..0322e18 100644 --- a/euscanwww/djeuscan/admin.py +++ b/euscanwww/djeuscan/admin.py @@ -7,7 +7,7 @@ from django.contrib import admin class EuscanResultAdmin(admin.ModelAdmin): search_fields = ('package__name', 'package__category') - list_filter = ('datetime', ) + list_filter = ('datetime', 'package__category') ordering = ["-datetime"] @@ -31,6 +31,12 @@ class VersionAdmin(admin.ModelAdmin): list_filter = ('overlay', 'packaged', 'alive') +class ProblemReportAdmin(admin.ModelAdmin): + list_display = ('package', 'subject', 'datetime') + search_fields = ('package__name', 'package__category') + list_filter = ('datetime', 'package__category') + ordering = ["-datetime"] + admin.site.register(Package, PackageAdmin) admin.site.register(Herd, HerdAdmin) @@ -52,4 +58,5 @@ admin.site.register(HerdAssociation) admin.site.register(CategoryAssociation) admin.site.register(MaintainerAssociation) admin.site.register(PackageAssociation) -admin.site.register(ProblemReport) + +admin.site.register(ProblemReport, ProblemReportAdmin)