diff --git a/bin/euscan b/bin/euscan index e0371b4..441d479 100755 --- a/bin/euscan +++ b/bin/euscan @@ -5,7 +5,7 @@ Distributed under the terms of the GNU General Public License v2 """ from __future__ import print_function - +import os # Meta @@ -19,7 +19,6 @@ __description__ = "A tool to detect new upstream releases." # Imports import sys -import os import getopt from errno import EINTR, EINVAL from httplib import HTTPConnection diff --git a/euscanwww/djeuscan/admin.py b/euscanwww/djeuscan/admin.py index a64007d..0322e18 100644 --- a/euscanwww/djeuscan/admin.py +++ b/euscanwww/djeuscan/admin.py @@ -7,7 +7,7 @@ from django.contrib import admin class EuscanResultAdmin(admin.ModelAdmin): search_fields = ('package__name', 'package__category') - list_filter = ('datetime', ) + list_filter = ('datetime', 'package__category') ordering = ["-datetime"] @@ -31,6 +31,12 @@ class VersionAdmin(admin.ModelAdmin): list_filter = ('overlay', 'packaged', 'alive') +class ProblemReportAdmin(admin.ModelAdmin): + list_display = ('package', 'subject', 'datetime') + search_fields = ('package__name', 'package__category') + list_filter = ('datetime', 'package__category') + ordering = ["-datetime"] + admin.site.register(Package, PackageAdmin) admin.site.register(Herd, HerdAdmin) @@ -52,4 +58,5 @@ admin.site.register(HerdAssociation) admin.site.register(CategoryAssociation) admin.site.register(MaintainerAssociation) admin.site.register(PackageAssociation) -admin.site.register(ProblemReport) + +admin.site.register(ProblemReport, ProblemReportAdmin) diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py index 0c38909..6626a41 100644 --- a/euscanwww/djeuscan/models.py +++ b/euscanwww/djeuscan/models.py @@ -1,3 +1,5 @@ +import json + from django.db import models from django.core.validators import RegexValidator, validate_email, URLValidator from django.core.exceptions import ValidationError @@ -215,6 +217,14 @@ class EuscanResult(models.Model): self.full_clean() super(EuscanResult, self).save(*args, **kwargs) + def messages(self): + result = json.loads(self.result) + + if result and self.package.cp() in result: + return result[self.package.cp()]['messages'] + else: + return "" + def __unicode__(self): return '[%s] %s/%s' % ( self.datetime, self.package.category, self.package.name diff --git a/euscanwww/djeuscan/templates/euscan/_package_details.html b/euscanwww/djeuscan/templates/euscan/_package_details.html new file mode 100644 index 0000000..1e04911 --- /dev/null +++ b/euscanwww/djeuscan/templates/euscan/_package_details.html @@ -0,0 +1,122 @@ +{% load djeuscan_helpers %} +{% load url from future %} + +
+ {% if package.description %} +
Description
+
{{ package.description }}
+ {% endif %} + {% if package.homepage %} +
Homepage
+
+ {% for homepage in package.homepages %} + {{ homepage }}
+ {% endfor %} +
+ {% endif %} + {% if package.herds.all %} +
Herds
+
+ {% for herd in package.herds.all %} + + {{ herd.herd }} + + <{{ herd.email }}> + {% endfor %} +
+ {% endif %} + {% if package.maintainers.all %} +
Maintainers
+
+ {% for maintainer in package.maintainers.all %} + {% if maintainer.name != maintainer.email %} + + {{ maintainer.name }} + + <{{ maintainer.email }}> + {% else %} + + {{ maintainer.email }} + + {% endif %} + {% endfor %} +
+ {% endif %} + {% if packaged %} +
Packaged Versions
+
+ +
+ {% endif %} + {% if upstream %} +
Upstream versions
+
+ +
+ {% endif %} + {% if vlog %} +
Version history
+
+ +
+ {% endif %} + {% if log %} +
euscan log
+
+

Date: {{ log.datetime }} + 

{{ msg|ansi_to_html|safe }}
+
+ {% endif %} +
diff --git a/euscanwww/djeuscan/templates/euscan/package.html b/euscanwww/djeuscan/templates/euscan/package.html index 2357318..8c90efe 100644 --- a/euscanwww/djeuscan/templates/euscan/package.html +++ b/euscanwww/djeuscan/templates/euscan/package.html @@ -1,7 +1,6 @@ {% extends "euscan/_datatable.html" %} {% load sub %} -{% load djeuscan_helpers %} {% load url from future %} {% block meta %} @@ -26,12 +25,6 @@ {% block content %} -{% if thanks_for_reporting %} -
- Thanks! Your report has been sent to admins -
-{% endif %} -
A refresh request is in progress, please wait...
@@ -56,158 +49,18 @@ {% endif %} -
- {% if package.description %} -
Description
-
{{ package.description }}
- {% endif %} - {% if package.homepage %} -
Homepage
-
- {% for homepage in package.homepages %} - {{ homepage }}
- {% endfor %} -
- {% endif %} - {% if package.herds.all %} -
Herds
-
- {% for herd in package.herds.all %} - - {{ herd.herd }} - - <{{ herd.email }}> - {% endfor %} -
- {% endif %} - {% if package.maintainers.all %} -
Maintainers
-
- {% for maintainer in package.maintainers.all %} - {% if maintainer.name != maintainer.email %} - - {{ maintainer.name }} - - <{{ maintainer.email }}> - {% else %} - - {{ maintainer.email }} - - {% endif %} - {% endfor %} -
- {% endif %} - {% if packaged %} -
Packaged Versions
-
- -
- {% endif %} - {% if upstream %} -
Upstream versions
-
- -
- {% endif %} -
Version history
-
- - - {% if log %} -
euscan log
-
-

Date: {{ log.datetime }} - 

{{ msg|ansi_to_html|safe }}
-
- {% endif %} -
+{% include "euscan/_package_details.html" %}
-

Report problems

-
-
-
- -
- {{ problem_form.version }} - {{ problem_form.version.errors.as_text }} -
-
-
- -
- {{ problem_form.subject }} - {{ problem_form.subject.errors.as_text }} -
-
-
- -
- {{ problem_form.message }} - {{ problem_form.message.errors.as_text }} -
-
- -
-
- -
-
-
-
+{% if user.is_authenticated %} +
+ +
+{% endif %} diff --git a/euscanwww/djeuscan/templates/euscan/problem.html b/euscanwww/djeuscan/templates/euscan/problem.html new file mode 100644 index 0000000..828b7ab --- /dev/null +++ b/euscanwww/djeuscan/templates/euscan/problem.html @@ -0,0 +1,57 @@ +{% extends "euscan/_datatable.html" %} + +{% load sub %} +{% load djeuscan_helpers %} +{% load url from future %} + +{% block title %} +{{ block.super }} - Report Problem: {{ package.category }}/{{ package.name }} +{% endblock %} + +{% block content %} + +{% if thanks_for_reporting %} +
+ Thanks! Your report has been sent to admins +
+{% endif %} + +

+ Report Problem: {{ package.category }}/{{ package.name }} +

+ +{% include "euscan/_package_details.html" %} +
+
+
+
+ +
+ {{ form.version }} + {{ form.version.errors.as_text }} +
+
+
+ +
+ {{ form.subject }} + {{ form.subject.errors.as_text }} +
+
+
+ +
+ {{ form.message }} + {{ form.message.errors.as_text }} +
+
+ +
+
+ +
+
+
+
+ +{% endblock %} diff --git a/euscanwww/djeuscan/urls.py b/euscanwww/djeuscan/urls.py index e4611d7..b1b1f88 100644 --- a/euscanwww/djeuscan/urls.py +++ b/euscanwww/djeuscan/urls.py @@ -22,6 +22,8 @@ package_patterns = patterns('djeuscan.views', 'unfavourite/$'), 'unfavourite_package', name="unfavourite_package"), url((r'^(?P[\w+][\w+.-]*)/(?P[\w+][\w+.-]*)/' 'refresh$'), "refresh_package", name="refresh_package"), + url(r'^(?P[\w+][\w+.-]*)/(?P[\w+][\w+.-]*)/problem$', + 'problem', name="problem"), ) categories_patterns = patterns('djeuscan.views', diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py index 3d96cb0..68b3d07 100644 --- a/euscanwww/djeuscan/views.py +++ b/euscanwww/djeuscan/views.py @@ -1,7 +1,6 @@ """ Views """ import inspect -import json from annoying.decorators import render_to, ajax_request from django.http import Http404 @@ -203,13 +202,6 @@ def package(request, category, package): log = log[0] if log else None vlog = VersionLog.objects.for_package(package, order=True) - result = json.loads(log.result) if log else None - - if result and package.cp() in result: - msg = result[package.cp()]['messages'] - else: - msg = "" - try: last_scan = EuscanResult.objects.for_package(package).latest().datetime except EuscanResult.DoesNotExist: @@ -230,32 +222,52 @@ def package(request, category, package): except RefreshPackageQuery.DoesNotExist: refreshed = False - thanks_for_reporting = False - if request.method == "POST": - problem_form = ProblemReportForm(package, request.POST) - if problem_form.is_valid(): - ProblemReport( - package=package, - version=problem_form.cleaned_data["version"], - subject=problem_form.cleaned_data["subject"], - message=problem_form.cleaned_data["message"], - ).save() - thanks_for_reporting = True - else: - problem_form = ProblemReportForm(package) - return { 'package': package, 'packaged': packaged, 'upstream': upstream, - 'log': log, + 'log': log.messages(), 'vlog': vlog, - 'msg': msg, + 'msg': log.messages() if log else "", 'last_scan': last_scan, 'favourited': favourited, 'refreshed': refreshed, - 'problem_form': problem_form, - 'thanks_for_reporting': thanks_for_reporting + } + + +@login_required +@render_to('euscan/problem.html') +def problem(request, category, package): + package = get_object_or_404(Package, category=category, name=package) + packaged = Version.objects.filter(package=package, packaged=True) + upstream = Version.objects.filter(package=package, packaged=False) + + log = EuscanResult.objects.filter(package=package).\ + order_by('-datetime')[:1] + log = log[0] if log else None + + thanks_for_reporting = False + + if request.method == "POST": + form = ProblemReportForm(package, request.POST) + if form.is_valid(): + ProblemReport( + package=package, + version=form.cleaned_data["version"], + subject=form.cleaned_data["subject"], + message=form.cleaned_data["message"], + ).save() + thanks_for_reporting = True + else: + form = ProblemReportForm(package) + + return { + 'form': form, + 'thanks_for_reporting': thanks_for_reporting, + 'package': package, + 'packaged': packaged, + 'upstream': upstream, + 'msg': log.messages() if log else "", } @@ -264,8 +276,10 @@ def world(request): world_form = WorldForm() packages_form = PackagesForm() - return {'world_form': world_form, - 'packages_form': packages_form} + return { + 'world_form': world_form, + 'packages_form': packages_form + } @render_to('euscan/world_scan.html') diff --git a/euscanwww/htdocs/img/bug.png b/euscanwww/htdocs/img/bug.png new file mode 100644 index 0000000..c33bfb9 Binary files /dev/null and b/euscanwww/htdocs/img/bug.png differ diff --git a/pym/euscan/handlers/watch.py b/pym/euscan/handlers/watch.py new file mode 100644 index 0000000..a129281 --- /dev/null +++ b/pym/euscan/handlers/watch.py @@ -0,0 +1,143 @@ +import re +import urllib2 + +import portage + +from euscan.handlers import generic +from euscan import output, helpers + +PRIORITY = 100 + +HANDLER_NAME = "watch" +CONFIDENCE = 100.0 + + +is_pattern = r"\([^\/]+\)" + + +def can_handle(pkg, url): + try: + return pkg.metadata._xml_tree.find("upstream").find("watch") \ + is not None + except AttributeError: + return False + + +def parse_mangles(mangles, string): + for mangle in mangles: + # convert regex from perl format to python format + # there are some regex in this format: s/pattern/replacement/ + m = re.match(r"s/(.*[^\\])/(.*)/", mangle) + if not m: + # or in this format s|pattern|replacement| + m = re.match(r"s\|(.*[^\\])\|(.*)\|", mangle) + pattern, repl = m.groups() + repl = re.sub(r"\$(\d+)", r"\\\1", repl) + string = re.sub(pattern, repl, string) + return string + + +def clean_results(results, versionmangle, urlmangle): + ret = [] + + for path, version, _, _ in results: + version = parse_mangles(versionmangle, version) + path = parse_mangles(urlmangle, path) + ret.append((path, version, HANDLER_NAME, CONFIDENCE)) + + return ret + + +def parse_watch(pkg): + for watch_tag in pkg.metadata._xml_tree.find("upstream").findall("watch"): + try: + base, file_pattern = watch_tag.text.split(" ")[:2] + except ValueError: + base, file_pattern = watch_tag.text, None + + # the file pattern can be in the base url + pattern_regex = r"/([^/]*\([^/]*\)[^/]*)$" + match = re.search(pattern_regex, base) + if match: + file_pattern = match.group(1) + base = base.replace(file_pattern, "") + + # handle sf.net specially + base = base.replace( + "http://sf.net/", "http://qa.debian.org/watch/sf.php/" + ) + + vmangle = watch_tag.attrib.get("uversionmangle", None) or \ + watch_tag.attrib.get("versionmangle", None) + versionmangle = vmangle.split(";") if vmangle else [] + + umangle = watch_tag.attrib.get("downloadurlmangle", None) + urlmangle = umangle.split(";") if umangle else [] + + yield (base, file_pattern, versionmangle, urlmangle) + + +def handle_directory_patterns(base, file_pattern): + """ + Directory pattern matching + e.g.: base: ftp://ftp.nessus.org/pub/nessus/nessus-([\d\.]+)/src/ + file_pattern: nessus-core-([\d\.]+)\.tar\.gz + """ + splitted = base.split("/") + i = 0 + basedir = [] + for elem in splitted: + if re.search(is_pattern, elem): + break + basedir.append(elem) + i += 1 + basedir = "/".join(basedir) + directory_pattern = splitted[i] + final = "/".join(splitted[i + 1:]) + + try: + fp = helpers.urlopen(basedir) + except urllib2.URLError: + return [] + except IOError: + return [] + + if not fp: + return [] + + data = fp.read() + + if basedir.startswith("ftp://"): + scan_data = generic.scan_ftp(data, basedir, directory_pattern) + else: + scan_data = generic.scan_html(data, basedir, directory_pattern) + + return [("/".join((basedir, path, final)), file_pattern) + for _, path in scan_data] + + +def scan(pkg, url): + output.einfo("Using watch data") + + cp, ver, rev = portage.pkgsplit(pkg.cpv) + + results = [] + for base, file_pattern, versionmangle, urlmangle in parse_watch(pkg): + if not re.search(is_pattern, base): + steps = [(base, file_pattern)] + res = generic.scan_directory_recursive( + cp, ver, rev, "", steps, url + ) + else: + res = [] + for step in handle_directory_patterns(base, file_pattern): + res += generic.scan_directory_recursive( + cp, ver, rev, "", [step], url + ) + + results += clean_results(res, versionmangle, urlmangle) + return results + + +def brute_force(pkg, url): + return [] diff --git a/setup.py b/setup.py index 85856aa..9b7179f 100755 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ packages = [ ] tests_require = [ - 'factory-boy==1.1.3', + 'factory-boy>=1.1.3', ] setup( @@ -88,10 +88,13 @@ setup( ('master' if __version__ == '9999' else ('euscan-%s' % __version__)) ), install_requires=[ - 'Django==1.4', 'django-annoying==0.7.6', 'South==0.7.4', - 'django-piston==0.2.3', 'BeautifulSoup==3.2.1', 'matplotlib==1.1.0', - 'django-celery==3.0.1', 'django-registration==0.8', - 'python-ldap==2.4.10', 'django-auth-ldap==1.1', + # Command line utility + 'BeautifulSoup>=3.2.1', + # Web interface + 'Django>=1.4', 'django-annoying>=0.7.6', 'South>=0.7', + 'django-piston>=0.2.3', 'matplotlib>=1.1.0', + 'django-celery>=3.0.1', 'django-registration>=0.8', + 'python-ldap>=2.4.10', 'django-auth-ldap>=1.1', ], package_dir={'': 'pym'}, packages=packages,