Conflicts:
	bin/euscan_patch_metadata
	euscanwww/djeuscan/views.py
	pym/euscan/handlers/__init__.py
	pym/euscan/handlers/kde.py
	pym/euscan/handlers/url/cpan.py
	pym/euscan/handlers/url/generic.py
	pym/euscan/handlers/url/github.py
	pym/euscan/handlers/url/php.py
	pym/euscan/handlers/url/pypi.py
	pym/euscan/handlers/url/rubygem.py
	pym/euscan/scan.py

Signed-off-by: volpino <fox91@anche.no>
This commit is contained in:
volpino 2012-07-26 10:57:11 +02:00
commit 3ef800b68d
11 changed files with 402 additions and 196 deletions

View File

@ -5,7 +5,7 @@ Distributed under the terms of the GNU General Public License v2
""" """
from __future__ import print_function from __future__ import print_function
import os
# Meta # Meta
@ -19,7 +19,6 @@ __description__ = "A tool to detect new upstream releases."
# Imports # Imports
import sys import sys
import os
import getopt import getopt
from errno import EINTR, EINVAL from errno import EINTR, EINVAL
from httplib import HTTPConnection from httplib import HTTPConnection

View File

@ -7,7 +7,7 @@ from django.contrib import admin
class EuscanResultAdmin(admin.ModelAdmin): class EuscanResultAdmin(admin.ModelAdmin):
search_fields = ('package__name', 'package__category') search_fields = ('package__name', 'package__category')
list_filter = ('datetime', ) list_filter = ('datetime', 'package__category')
ordering = ["-datetime"] ordering = ["-datetime"]
@ -31,6 +31,12 @@ class VersionAdmin(admin.ModelAdmin):
list_filter = ('overlay', 'packaged', 'alive') list_filter = ('overlay', 'packaged', 'alive')
class ProblemReportAdmin(admin.ModelAdmin):
list_display = ('package', 'subject', 'datetime')
search_fields = ('package__name', 'package__category')
list_filter = ('datetime', 'package__category')
ordering = ["-datetime"]
admin.site.register(Package, PackageAdmin) admin.site.register(Package, PackageAdmin)
admin.site.register(Herd, HerdAdmin) admin.site.register(Herd, HerdAdmin)
@ -52,4 +58,5 @@ admin.site.register(HerdAssociation)
admin.site.register(CategoryAssociation) admin.site.register(CategoryAssociation)
admin.site.register(MaintainerAssociation) admin.site.register(MaintainerAssociation)
admin.site.register(PackageAssociation) admin.site.register(PackageAssociation)
admin.site.register(ProblemReport)
admin.site.register(ProblemReport, ProblemReportAdmin)

View File

@ -1,3 +1,5 @@
import json
from django.db import models from django.db import models
from django.core.validators import RegexValidator, validate_email, URLValidator from django.core.validators import RegexValidator, validate_email, URLValidator
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
@ -215,6 +217,14 @@ class EuscanResult(models.Model):
self.full_clean() self.full_clean()
super(EuscanResult, self).save(*args, **kwargs) super(EuscanResult, self).save(*args, **kwargs)
def messages(self):
result = json.loads(self.result)
if result and self.package.cp() in result:
return result[self.package.cp()]['messages']
else:
return ""
def __unicode__(self): def __unicode__(self):
return '[%s] %s/%s' % ( return '[%s] %s/%s' % (
self.datetime, self.package.category, self.package.name self.datetime, self.package.category, self.package.name

View File

@ -0,0 +1,122 @@
{% load djeuscan_helpers %}
{% load url from future %}
<dl>
{% if package.description %}
<dt>Description</dt>
<dd>{{ package.description }}</dd>
{% endif %}
{% if package.homepage %}
<dt>Homepage</dt>
<dd>
{% for homepage in package.homepages %}
<a href="{{ homepage }}">{{ homepage }}</a><br />
{% endfor %}
</dd>
{% endif %}
{% if package.herds.all %}
<dt>Herds</dt>
<dd>
{% for herd in package.herds.all %}
<a href="{% url "herd" herd.herd %}">
{{ herd.herd }}
</a>
&lt;{{ herd.email }}&gt;
{% endfor %}
</dd>
{% endif %}
{% if package.maintainers.all %}
<dt>Maintainers</dt>
<dd>
{% for maintainer in package.maintainers.all %}
{% if maintainer.name != maintainer.email %}
<a href="{% url "maintainer" maintainer.id %}">
{{ maintainer.name }}
</a>
&lt;{{ maintainer.email }}&gt;
{% else %}
<a href="{% url "maintainer" maintainer.id %}">
{{ maintainer.email }}
</a>
{% endif %}
{% endfor %}
</dd>
{% endif %}
{% if packaged %}
<dt>Packaged Versions</dt>
<dd>
<ul>
{% for version in packaged %}
<li id="{{ version.version }}-{{version.revision }}:{{ version.slot }}-[{{ version.overlay }}]">
{% if version.overlay == "gentoo" %}
<img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
{% else %}
<img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
{% endif %}
{% if version.vtype and not version.vtype|is_stable %}
<img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" />
{% endif %}
{{ version.version }}-{{ version.revision }} :{{ version.slot }} [{{ version.overlay }}]
</li>
{% endfor %}
</ul>
</dd>
{% endif %}
{% if upstream %}
<dt>Upstream versions</dt>
<dd>
<ul>
{% for version in upstream %}
<li>
<img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
{% if version.vtype and not version.vtype|is_stable %}
<img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" />
{% endif %}
{{ version.version }} - {{ version.urls }}
{% if confidence < 100 %}({{ version.confidence }}%){% endif %}
</li>
{% endfor %}
</ul>
</dd>
{% endif %}
{% if vlog %}
<dt>Version history</dt>
<dd>
<ul class="log">
{% for version in vlog %}
{% if version.action == version.VERSION_ADDED %}
<li class="added">
{% else %}
<li class="removed">
{% endif %}
{% if version.overlay == "gentoo" %}
<img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
{% elif version.overlay %}
<img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
{% else %}
<img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
{% endif %}
{% if version.vtype and not version.vtype|is_stable %}
<img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" />
{% endif %}
{{ version }} - {{ version.datetime }}
</li>
{% endfor %}
</ul>
</dd>
{% endif %}
{% if log %}
<dt>euscan log</dt>
<dd>
<p>Date: {{ log.datetime }}
<pre class="log">{{ msg|ansi_to_html|safe }}</pre>
</dd>
{% endif %}
</dl>

View File

@ -1,7 +1,6 @@
{% extends "euscan/_datatable.html" %} {% extends "euscan/_datatable.html" %}
{% load sub %} {% load sub %}
{% load djeuscan_helpers %}
{% load url from future %} {% load url from future %}
{% block meta %} {% block meta %}
@ -26,12 +25,6 @@
{% block content %} {% block content %}
{% if thanks_for_reporting %}
<div class="alert alert-success">
<strong>Thanks!</strong> Your report has been sent to admins
</div>
{% endif %}
<div class="refresh-alert alert {% if not refreshed %}hide{% endif %}"> <div class="refresh-alert alert {% if not refreshed %}hide{% endif %}">
A refresh request is in progress, please wait... A refresh request is in progress, please wait...
</div> </div>
@ -56,158 +49,18 @@
</span> </span>
{% endif %} {% endif %}
</h2> </h2>
<dl>
{% if package.description %}
<dt>Description</dt>
<dd>{{ package.description }}</dd>
{% endif %}
{% if package.homepage %}
<dt>Homepage</dt>
<dd>
{% for homepage in package.homepages %}
<a href="{{ homepage }}">{{ homepage }}</a><br />
{% endfor %}
</dd>
{% endif %}
{% if package.herds.all %}
<dt>Herds</dt>
<dd>
{% for herd in package.herds.all %}
<a href="{% url "herd" herd.herd %}">
{{ herd.herd }}
</a>
&lt;{{ herd.email }}&gt;
{% endfor %}
</dd>
{% endif %}
{% if package.maintainers.all %}
<dt>Maintainers</dt>
<dd>
{% for maintainer in package.maintainers.all %}
{% if maintainer.name != maintainer.email %}
<a href="{% url "maintainer" maintainer.id %}">
{{ maintainer.name }}
</a>
&lt;{{ maintainer.email }}&gt;
{% else %}
<a href="{% url "maintainer" maintainer.id %}">
{{ maintainer.email }}
</a>
{% endif %}
{% endfor %}
</dd>
{% endif %}
{% if packaged %}
<dt>Packaged Versions</dt>
<dd>
<ul>
{% for version in packaged %}
<li id="{{ version.version }}-{{version.revision }}:{{ version.slot }}-[{{ version.overlay }}]">
{% if version.overlay == "gentoo" %}
<img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
{% else %}
<img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
{% endif %}
{% if version.vtype and not version.vtype|is_stable %} {% include "euscan/_package_details.html" %}
<img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" />
{% endif %}
{{ version.version }}-{{ version.revision }} :{{ version.slot }} [{{ version.overlay }}]
</li>
{% endfor %}
</ul>
</dd>
{% endif %}
{% if upstream %}
<dt>Upstream versions</dt>
<dd>
<ul>
{% for version in upstream %}
<li>
<img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
{% if version.vtype and not version.vtype|is_stable %}
<img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" />
{% endif %}
{{ version.version }} - {{ version.urls }}
{% if confidence < 100 %}({{ version.confidence }}%){% endif %}
</li>
{% endfor %}
</ul>
</dd>
{% endif %}
<dt>Version history</dt>
<dd>
<ul class="log">
{% for version in vlog %}
{% if version.action == version.VERSION_ADDED %}
<li class="added">
{% else %}
<li class="removed">
{% endif %}
{% if version.overlay == "gentoo" %}
<img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
{% elif version.overlay %}
<img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
{% else %}
<img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
{% endif %}
{% if version.vtype and not version.vtype|is_stable %}
<img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" />
{% endif %}
{{ version }} - {{ version.datetime }}
</li>
{% endfor %}
</ul>
</dt>
{% if log %}
<dt>euscan log</dt>
<dd>
<p>Date: {{ log.datetime }}
<pre class="log">{{ msg|ansi_to_html|safe }}</pre>
</dd>
{% endif %}
</dl>
<hr/> <hr/>
<h3 class="report-problems-toggle cursor-pointer">Report problems</h3> {% if user.is_authenticated %}
<div class="report-problems hide"> <form class="pull-right" action="{% url "problem" package.category package.name %}" method="get">
<form action="." method="post" class="form-horizontal"> <button class="btn bug-button">
<div class="control-group {% if problem_form.version.errors %}error{% endif %}"> <img src="{{ STATIC_URL}}/img/bug.png" alt="Report problems" >
<label class="control-label" for="version">{{ problem_form.version.label }}</label> </button>
<div class="controls"> </form>
{{ problem_form.version }} {% endif %}
<span class="help-inline">{{ problem_form.version.errors.as_text }}</span>
</div>
</div>
<div class="control-group {% if problem_form.subject.errors %}error{% endif %}">
<label class="control-label" for="subject">{{ problem_form.subject.label }}</label>
<div class="controls">
{{ problem_form.subject }}
<span class="help-inline">{{ problem_form.subject.errors.as_text }}</span>
</div>
</div>
<div class="control-group {% if problem_form.message.errors %}error{% endif %}">
<label class="control-label" for="message">{{ problem_form.message.label }}</label>
<div class="controls">
{{ problem_form.message }}
<span class="help-inline">{{ problem_form.message.errors.as_text }}</span>
</div>
</div>
<div class="control-group">
<div class="controls">
<input class="btn" type="submit" value="Submit" />
</div>
</div>
</form>
</div>
<script type="text/javascript"> <script type="text/javascript">
$(document).ready(function () { $(document).ready(function () {
@ -235,10 +88,6 @@
}); });
}); });
$(".report-problems-toggle").click(function () {
$(".report-problems").toggle("slow");
});
}); });
</script> </script>

View File

@ -0,0 +1,57 @@
{% extends "euscan/_datatable.html" %}
{% load sub %}
{% load djeuscan_helpers %}
{% load url from future %}
{% block title %}
{{ block.super }} - Report Problem: {{ package.category }}/{{ package.name }}
{% endblock %}
{% block content %}
{% if thanks_for_reporting %}
<div class="alert alert-success">
<strong>Thanks!</strong> Your report has been sent to admins
</div>
{% endif %}
<h2>
Report Problem: {{ package.category }}/{{ package.name }}
</h2>
{% include "euscan/_package_details.html" %}
<hr/>
<div class="report-problems">
<form action="{% url "problem" package.category package.name %}" method="post" class="form-horizontal">
<div class="control-group {% if form.version.errors %}error{% endif %}">
<label class="control-label" for="version">{{ form.version.label }}</label>
<div class="controls">
{{ form.version }}
<span class="help-inline">{{ form.version.errors.as_text }}</span>
</div>
</div>
<div class="control-group {% if form.subject.errors %}error{% endif %}">
<label class="control-label" for="subject">{{ form.subject.label }}</label>
<div class="controls">
{{ form.subject }}
<span class="help-inline">{{ form.subject.errors.as_text }}</span>
</div>
</div>
<div class="control-group {% if form.message.errors %}error{% endif %}">
<label class="control-label" for="message">{{ form.message.label }}</label>
<div class="controls">
{{ form.message }}
<span class="help-inline">{{ form.message.errors.as_text }}</span>
</div>
</div>
<div class="control-group">
<div class="controls">
<input class="btn" type="submit" value="Submit" />
</div>
</div>
</form>
</div>
{% endblock %}

View File

@ -22,6 +22,8 @@ package_patterns = patterns('djeuscan.views',
'unfavourite/$'), 'unfavourite_package', name="unfavourite_package"), 'unfavourite/$'), 'unfavourite_package', name="unfavourite_package"),
url((r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/' url((r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/'
'refresh$'), "refresh_package", name="refresh_package"), 'refresh$'), "refresh_package", name="refresh_package"),
url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/problem$',
'problem', name="problem"),
) )
categories_patterns = patterns('djeuscan.views', categories_patterns = patterns('djeuscan.views',

View File

@ -1,7 +1,6 @@
""" Views """ """ Views """
import inspect import inspect
import json
from annoying.decorators import render_to, ajax_request from annoying.decorators import render_to, ajax_request
from django.http import Http404 from django.http import Http404
@ -203,13 +202,6 @@ def package(request, category, package):
log = log[0] if log else None log = log[0] if log else None
vlog = VersionLog.objects.for_package(package, order=True) vlog = VersionLog.objects.for_package(package, order=True)
result = json.loads(log.result) if log else None
if result and package.cp() in result:
msg = result[package.cp()]['messages']
else:
msg = ""
try: try:
last_scan = EuscanResult.objects.for_package(package).latest().datetime last_scan = EuscanResult.objects.for_package(package).latest().datetime
except EuscanResult.DoesNotExist: except EuscanResult.DoesNotExist:
@ -230,32 +222,52 @@ def package(request, category, package):
except RefreshPackageQuery.DoesNotExist: except RefreshPackageQuery.DoesNotExist:
refreshed = False refreshed = False
thanks_for_reporting = False
if request.method == "POST":
problem_form = ProblemReportForm(package, request.POST)
if problem_form.is_valid():
ProblemReport(
package=package,
version=problem_form.cleaned_data["version"],
subject=problem_form.cleaned_data["subject"],
message=problem_form.cleaned_data["message"],
).save()
thanks_for_reporting = True
else:
problem_form = ProblemReportForm(package)
return { return {
'package': package, 'package': package,
'packaged': packaged, 'packaged': packaged,
'upstream': upstream, 'upstream': upstream,
'log': log, 'log': log.messages(),
'vlog': vlog, 'vlog': vlog,
'msg': msg, 'msg': log.messages() if log else "",
'last_scan': last_scan, 'last_scan': last_scan,
'favourited': favourited, 'favourited': favourited,
'refreshed': refreshed, 'refreshed': refreshed,
'problem_form': problem_form, }
'thanks_for_reporting': thanks_for_reporting
@login_required
@render_to('euscan/problem.html')
def problem(request, category, package):
package = get_object_or_404(Package, category=category, name=package)
packaged = Version.objects.filter(package=package, packaged=True)
upstream = Version.objects.filter(package=package, packaged=False)
log = EuscanResult.objects.filter(package=package).\
order_by('-datetime')[:1]
log = log[0] if log else None
thanks_for_reporting = False
if request.method == "POST":
form = ProblemReportForm(package, request.POST)
if form.is_valid():
ProblemReport(
package=package,
version=form.cleaned_data["version"],
subject=form.cleaned_data["subject"],
message=form.cleaned_data["message"],
).save()
thanks_for_reporting = True
else:
form = ProblemReportForm(package)
return {
'form': form,
'thanks_for_reporting': thanks_for_reporting,
'package': package,
'packaged': packaged,
'upstream': upstream,
'msg': log.messages() if log else "",
} }
@ -264,8 +276,10 @@ def world(request):
world_form = WorldForm() world_form = WorldForm()
packages_form = PackagesForm() packages_form = PackagesForm()
return {'world_form': world_form, return {
'packages_form': packages_form} 'world_form': world_form,
'packages_form': packages_form
}
@render_to('euscan/world_scan.html') @render_to('euscan/world_scan.html')

Binary file not shown.

After

Width:  |  Height:  |  Size: 925 B

View File

@ -0,0 +1,143 @@
import re
import urllib2
import portage
from euscan.handlers import generic
from euscan import output, helpers
PRIORITY = 100
HANDLER_NAME = "watch"
CONFIDENCE = 100.0
is_pattern = r"\([^\/]+\)"
def can_handle(pkg, url):
try:
return pkg.metadata._xml_tree.find("upstream").find("watch") \
is not None
except AttributeError:
return False
def parse_mangles(mangles, string):
for mangle in mangles:
# convert regex from perl format to python format
# there are some regex in this format: s/pattern/replacement/
m = re.match(r"s/(.*[^\\])/(.*)/", mangle)
if not m:
# or in this format s|pattern|replacement|
m = re.match(r"s\|(.*[^\\])\|(.*)\|", mangle)
pattern, repl = m.groups()
repl = re.sub(r"\$(\d+)", r"\\\1", repl)
string = re.sub(pattern, repl, string)
return string
def clean_results(results, versionmangle, urlmangle):
ret = []
for path, version, _, _ in results:
version = parse_mangles(versionmangle, version)
path = parse_mangles(urlmangle, path)
ret.append((path, version, HANDLER_NAME, CONFIDENCE))
return ret
def parse_watch(pkg):
for watch_tag in pkg.metadata._xml_tree.find("upstream").findall("watch"):
try:
base, file_pattern = watch_tag.text.split(" ")[:2]
except ValueError:
base, file_pattern = watch_tag.text, None
# the file pattern can be in the base url
pattern_regex = r"/([^/]*\([^/]*\)[^/]*)$"
match = re.search(pattern_regex, base)
if match:
file_pattern = match.group(1)
base = base.replace(file_pattern, "")
# handle sf.net specially
base = base.replace(
"http://sf.net/", "http://qa.debian.org/watch/sf.php/"
)
vmangle = watch_tag.attrib.get("uversionmangle", None) or \
watch_tag.attrib.get("versionmangle", None)
versionmangle = vmangle.split(";") if vmangle else []
umangle = watch_tag.attrib.get("downloadurlmangle", None)
urlmangle = umangle.split(";") if umangle else []
yield (base, file_pattern, versionmangle, urlmangle)
def handle_directory_patterns(base, file_pattern):
"""
Directory pattern matching
e.g.: base: ftp://ftp.nessus.org/pub/nessus/nessus-([\d\.]+)/src/
file_pattern: nessus-core-([\d\.]+)\.tar\.gz
"""
splitted = base.split("/")
i = 0
basedir = []
for elem in splitted:
if re.search(is_pattern, elem):
break
basedir.append(elem)
i += 1
basedir = "/".join(basedir)
directory_pattern = splitted[i]
final = "/".join(splitted[i + 1:])
try:
fp = helpers.urlopen(basedir)
except urllib2.URLError:
return []
except IOError:
return []
if not fp:
return []
data = fp.read()
if basedir.startswith("ftp://"):
scan_data = generic.scan_ftp(data, basedir, directory_pattern)
else:
scan_data = generic.scan_html(data, basedir, directory_pattern)
return [("/".join((basedir, path, final)), file_pattern)
for _, path in scan_data]
def scan(pkg, url):
output.einfo("Using watch data")
cp, ver, rev = portage.pkgsplit(pkg.cpv)
results = []
for base, file_pattern, versionmangle, urlmangle in parse_watch(pkg):
if not re.search(is_pattern, base):
steps = [(base, file_pattern)]
res = generic.scan_directory_recursive(
cp, ver, rev, "", steps, url
)
else:
res = []
for step in handle_directory_patterns(base, file_pattern):
res += generic.scan_directory_recursive(
cp, ver, rev, "", [step], url
)
results += clean_results(res, versionmangle, urlmangle)
return results
def brute_force(pkg, url):
return []

View File

@ -70,7 +70,7 @@ packages = [
] ]
tests_require = [ tests_require = [
'factory-boy==1.1.3', 'factory-boy>=1.1.3',
] ]
setup( setup(
@ -88,10 +88,13 @@ setup(
('master' if __version__ == '9999' else ('euscan-%s' % __version__)) ('master' if __version__ == '9999' else ('euscan-%s' % __version__))
), ),
install_requires=[ install_requires=[
'Django==1.4', 'django-annoying==0.7.6', 'South==0.7.4', # Command line utility
'django-piston==0.2.3', 'BeautifulSoup==3.2.1', 'matplotlib==1.1.0', 'BeautifulSoup>=3.2.1',
'django-celery==3.0.1', 'django-registration==0.8', # Web interface
'python-ldap==2.4.10', 'django-auth-ldap==1.1', 'Django>=1.4', 'django-annoying>=0.7.6', 'South>=0.7',
'django-piston>=0.2.3', 'matplotlib>=1.1.0',
'django-celery>=3.0.1', 'django-registration>=0.8',
'python-ldap>=2.4.10', 'django-auth-ldap>=1.1',
], ],
package_dir={'': 'pym'}, package_dir={'': 'pym'},
packages=packages, packages=packages,