From 9afb31fc65934e387cf93efb2b275946fa254e91 Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Sat, 15 Sep 2012 23:20:30 +0200 Subject: [PATCH 01/19] euscan: fix #435118 and #435120 Signed-off-by: Corentin Chary --- bin/euscan | 4 ++-- pym/euscan/__init__.py | 11 ++++++----- pym/euscan/handlers/generic.py | 1 + pym/euscan/helpers.py | 27 ++++++++++++++------------- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/bin/euscan b/bin/euscan index 5d5e812..79053e5 100755 --- a/bin/euscan +++ b/bin/euscan @@ -287,9 +287,9 @@ def main(): from os.path import basename # To get the short name - output.error( + output.eerror( "The short ebuild name '%s' is ambiguous. Please specify" % - basename(pkgs[0]), + basename(pkgs[0]) + "one of the above fully-qualified ebuild names instead." ) exit_helper(1) diff --git a/pym/euscan/__init__.py b/pym/euscan/__init__.py index 12c4a16..946c63c 100644 --- a/pym/euscan/__init__.py +++ b/pym/euscan/__init__.py @@ -77,11 +77,12 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [ '(.*)sourceforge(.*)', '(.*)github.com', '(.*)berlios(.*)', - '(.*)qt.nokia.com(.*)', - '(.*)chromium.org(.*)', - '(.*)nodejs.org(.*)', - '(.*)download.mono-project.com(.*)', - '(.*)fedorahosted.org(.*)', + '(.*)qt\.nokia\.com(.*)', + '(.*)chromium\.org(.*)', + '(.*)nodejs\.org(.*)', + '(.*)download\.mono-project\.com(.*)', + '(.*)fedorahosted\.org(.*)', + '(.*)download\.tuxfamily\.org(.*)', ] from out import EuscanOutput diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/generic.py index fd82c71..0795488 100644 --- a/pym/euscan/handlers/generic.py +++ b/pym/euscan/handlers/generic.py @@ -70,6 +70,7 @@ def scan_html(data, url, pattern): match.group(0)) ) + return results diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py index 5e3e6ea..3271811 100644 --- a/pym/euscan/helpers.py +++ b/pym/euscan/helpers.py @@ -373,27 +373,28 @@ def tryurl(fileurl, template): def regex_from_template(template): # Escape - template = re.escape(template) + regexp = re.escape(template) # Unescape specific stuff - template = template.replace('\$\{', '${') - template = template.replace('\}', '}') - template = template.replace('}\.$', '}.$') + regexp = regexp.replace('\$\{', '${') + regexp = regexp.replace('\}', '}') + regexp = regexp.replace('}\.$', '}.$') # Replace ${\d+} - #template = template.replace('${0}', r'([\d]+?)') - template = re.sub(r'(\$\{\d+\}(\.?))+', r'([\w\.]+?)', template) + #regexp = regexp.replace('${0}', r'([\d]+?)') + regexp = re.sub(r'(\$\{\d+\}(\.?))+', r'([\w\.]+?)', regexp) - #template = re.sub(r'(\$\{\d+\}\.?)+', r'([\w]+?)', template) - #template = re.sub(r'(\$\{\d+\}\.+)+', '(.+?)\.', template) - #template = re.sub(r'(\$\{\d+\})+', '(.+?)', template) + #regexp = re.sub(r'(\$\{\d+\}\.?)+', r'([\w]+?)', regexp) + #regexp = re.sub(r'(\$\{\d+\}\.+)+', '(.+?)\.', regexp) + #regexp = re.sub(r'(\$\{\d+\})+', '(.+?)', regexp) # Full version - template = template.replace('${PV}', _v) + regexp = regexp.replace('${PV}', _v) # End - template = template + r'/?$' - return template + regexp = regexp + r'/?$' + + return regexp def basedir_from_template(template): @@ -417,7 +418,7 @@ def generate_scan_paths(url): path = prefix + ":/" for chunk in chunks: if '${' in chunk: - steps.append((path, regex_from_template(chunk))) + steps.append((path, '^(?:|.*/)' + regex_from_template(chunk))) path = "" else: path += "/" From 2bd6c03b3b3deee5a15a3011407a2319b8e57931 Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Wed, 10 Oct 2012 09:36:25 +0200 Subject: [PATCH 02/19] djeuscan: really fix scan on demand don't use stupid locks, only rate limit. Signed-off-by: Corentin Chary --- euscanwww/djeuscan/tasks.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py index b9aa92b..5e931d2 100644 --- a/euscanwww/djeuscan/tasks.py +++ b/euscanwww/djeuscan/tasks.py @@ -80,7 +80,6 @@ def update_counters(fast=False): """ Updates counters """ - logger = update_counters.get_logger() logger.info("Updating counters (fast=%s)...", fast) misc.update_counters(fast=fast) @@ -235,20 +234,12 @@ def scan_package_user(package): return True -@task +@task(rate_limit="1/m") def consume_refresh_queue(locked=False): """ Satisfies user requests for package refreshing, runs every minute """ - LOCK_ID = 'lock-consume-refresh-queue' - unlock = lambda: cache.delete(LOCK_ID) - lock = lambda: cache.add(LOCK_ID, True, 120) - logger = consume_refresh_queue.get_logger() - - if not locked and not lock(): - return - logger.info('Consuming package refresh request queue...') try: @@ -258,17 +249,14 @@ def consume_refresh_queue(locked=False): scan_package_user.delay(pkg) logger.info('Selected: %s' % pkg) except IndexError: - pass - finally: - unlock() + return if RefreshPackageQuery.objects.count(): logger.info('Restarting myself in 60s') - lock() consume_refresh_queue.apply_async( kwargs={'locked': True}, countdown=60 ) - + return True @task(max_retries=10, default_retry_delay=10 * 60) def send_user_email(address, subject, text): @@ -279,7 +267,7 @@ def send_user_email(address, subject, text): ) except Exception, exc: raise send_user_email.retry(exc=exc) - + return True @task def process_emails(profiles, only_if_vlogs=False): @@ -321,7 +309,7 @@ def process_emails(profiles, only_if_vlogs=False): profile.last_email = now profile.save(force_update=True) - + return True @task def send_update_email(): @@ -335,6 +323,7 @@ def send_update_email(): settings.TASKS_EMAIL_GROUPS, only_if_vlogs=True )() + return True @task @@ -344,6 +333,7 @@ def send_weekly_email(): email_activated=True ) group_chunks(process_emails, profiles, settings.TASKS_EMAIL_GROUPS)() + return True @task @@ -353,7 +343,7 @@ def send_monthly_email(): email_activated=True ) group_chunks(process_emails, profiles, settings.TASKS_EMAIL_GROUPS)() - + return True admin_tasks = [ regen_rrds, From 60deecd8532361270bc7a212e27971fa7341589a Mon Sep 17 00:00:00 2001 From: volpino Date: Fri, 14 Sep 2012 11:33:57 +0200 Subject: [PATCH 03/19] euscan: Adding support for disabling handlers Signed-off-by: volpino --- TODO | 1 - bin/euscan | 7 ++++++- pym/euscan/__init__.py | 4 +++- pym/euscan/handlers/__init__.py | 10 +++++++--- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/TODO b/TODO index 8c7cf35..aad1854 100644 --- a/TODO +++ b/TODO @@ -9,7 +9,6 @@ euscan - Check other distros (youri, distrowatch, distromatch, whoas; Equivalent-Packages) - Steal ideas from other tools (uscan, portscout) - Steal data from other tools (dehs) -- Add a way to enable/disable handlers (--no-handler-kde --no-handler-sourceforge) ### Command line interface diff --git a/bin/euscan b/bin/euscan index 79053e5..255151b 100755 --- a/bin/euscan +++ b/bin/euscan @@ -133,6 +133,9 @@ def print_usage(_error=None, help=None): " - use mirror:// URIs", file=out) print(yellow(" --ebuild-uri") + " - use ebuild variables in URIs", file=out) + print(yellow(" --no-handlers") + + " - exclude handlers (comma-separated list)", + file=out) print(file=out) if _error in ('packages',) or help: @@ -194,6 +197,8 @@ def parse_args(): CONFIG['ignore-pre-release-if-stable'] = True elif o in ("--ebuild-uri"): CONFIG['ebuild-uri'] = True + elif o in ("--no-handlers"): + CONFIG['handlers-exclude'] = a.split(",") else: return_code = False @@ -205,7 +210,7 @@ def parse_args(): getopt_options['long']['global'] = [ "help", "version", "nocolor", "quiet", "verbose", "oneshot", "brute-force=", "format=", "progress", "mirror", "ignore-pre-release", - "ignore-pre-release-if-stable", "ebuild-uri" + "ignore-pre-release-if-stable", "ebuild-uri", "no-handlers=" ] short_opts = getopt_options['short']['global'] diff --git a/pym/euscan/__init__.py b/pym/euscan/__init__.py index 946c63c..49241dd 100644 --- a/pym/euscan/__init__.py +++ b/pym/euscan/__init__.py @@ -7,6 +7,7 @@ __version__ = "git" import ConfigParser import os +from ast import literal_eval CONFIG = { @@ -29,6 +30,7 @@ CONFIG = { 'ignore-pre-release': False, 'ignore-pre-release-if-stable': False, 'ebuild-uri': False, + 'handlers-exclude': [], } config = ConfigParser.ConfigParser() @@ -36,7 +38,7 @@ config.read(['/etc/euscan.conf', os.path.expanduser('~/.euscan.conf')]) if config.has_section("euscan"): for key, value in config.items("euscan"): if key in CONFIG: - CONFIG[key] = value + CONFIG[key] = literal_eval(value) BLACKLIST_VERSIONS = [ # Compatibility package for running binaries linked against a diff --git a/pym/euscan/handlers/__init__.py b/pym/euscan/handlers/__init__.py index ee4dedf..e89a0fc 100644 --- a/pym/euscan/handlers/__init__.py +++ b/pym/euscan/handlers/__init__.py @@ -37,7 +37,8 @@ def find_best_handler(kind, pkg, *args): Find the best handler for the given package """ for handler in handlers[kind]: - if handler.can_handle(pkg, *args): + if (handler.HANDLER_NAME not in CONFIG["handlers-exclude"] and + handler.can_handle(pkg, *args)): return handler return None @@ -149,8 +150,11 @@ def scan_url(pkg, urls, options, on_progress=None): try: url_handler = find_best_handler('url', pkg, url) - for o in options: - versions += url_handler.scan_url(pkg, url, o) + if url_handler: + for o in options: + versions += url_handler.scan_url(pkg, url, o) + else: + output.eerror("Can't find a suitable handler!") except Exception as e: output.ewarn( "Handler failed: [%s] %s" % From 16e14bf79185c9a455269d7bfe27a8c5543c17f8 Mon Sep 17 00:00:00 2001 From: volpino Date: Fri, 14 Sep 2012 15:25:08 +0200 Subject: [PATCH 04/19] euscanwww: Created a feed for scan world with packages in GET Note: Check if passing parameters in GET is good, there's the risk of looong urls Signed-off-by: volpino --- TODO | 1 - euscanwww/djeuscan/feeds.py | 25 ++++++++++++++++++- .../djeuscan/templates/euscan/world_scan.html | 12 +++++++++ euscanwww/djeuscan/urls.py | 3 ++- 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/TODO b/TODO index aad1854..1cc313f 100644 --- a/TODO +++ b/TODO @@ -52,7 +52,6 @@ euscanwww --------- - Add an /about/config page that describe the current config (overlays, stuff in make.conf, euscan default settings, etc..) -- Create a rss for "scan world" + packages in GET - Always keep in db all found versions (when using an API only?). But don't display them if older than current packaged version, except maybe in the "upstream_version" column. ### packages diff --git a/euscanwww/djeuscan/feeds.py b/euscanwww/djeuscan/feeds.py index 5836d87..3455de5 100644 --- a/euscanwww/djeuscan/feeds.py +++ b/euscanwww/djeuscan/feeds.py @@ -229,9 +229,32 @@ class UserFeed(BaseFeed): } def _items(self, data): - user, options = data["user"], data["options"] + user = data["user"] profile = get_profile(user) vlogs = get_account_versionlogs(profile) return vlogs, 100 + + +class WorldScanFeed(BaseFeed): + link = "/" + + def get_object(self, request): + return { + "options": request.GET, + } + + def _items(self, data): + packages = [] + for pkg_name in data["options"].getlist("package", []): + if "/" in pkg_name: + cat, name = pkg_name.split("/", 1) + pkg = Package.objects.filter(name=name, category=cat) + else: + pkg = Package.objects.filter(name=pkg_name) + packages.extend(pkg) + + vlogs = VersionLog.objects.filter(package__in=packages) + + return vlogs, 100 diff --git a/euscanwww/djeuscan/templates/euscan/world_scan.html b/euscanwww/djeuscan/templates/euscan/world_scan.html index d579dc6..30a029e 100644 --- a/euscanwww/djeuscan/templates/euscan/world_scan.html +++ b/euscanwww/djeuscan/templates/euscan/world_scan.html @@ -7,6 +7,18 @@ {{ block.super }} - World Scan {% endblock %} +{% block menu_feed %} +{{ block.super }} +
  • + feed + + World scan feed + +
    • +{% endblock %} + + {% block content %}

    World scan: diff --git a/euscanwww/djeuscan/urls.py b/euscanwww/djeuscan/urls.py index 39893ee..e156a4b 100644 --- a/euscanwww/djeuscan/urls.py +++ b/euscanwww/djeuscan/urls.py @@ -8,7 +8,7 @@ from djcelery.views import apply as apply_task from djeuscan.views import registered_tasks from djeuscan.feeds import PackageFeed, CategoryFeed, HerdFeed, \ - MaintainerFeed, GlobalFeed, UserFeed + MaintainerFeed, GlobalFeed, UserFeed, WorldScanFeed admin_required = user_passes_test(lambda u: u.is_superuser) @@ -117,6 +117,7 @@ urlpatterns = patterns('djeuscan.views', url(r'^statistics/charts/(?P[\w\-]+).png$', 'chart', name="chart"), url(r'^world/$', 'world', name="world"), url(r'^world/scan/$', 'world_scan', name="world_scan"), + url(r'^world/scan/feed$', WorldScanFeed(), name="world_scan_feed"), url(r'^world/favourite/$', 'favourite_world', name="favourite_world"), url(r'^world/unfavourite/$', 'unfavourite_world', name="unfavourite_world"), From d2433c1b3c955fc86d3db8b2b1c6ea03a6e0e610 Mon Sep 17 00:00:00 2001 From: volpino Date: Fri, 14 Sep 2012 16:21:30 +0200 Subject: [PATCH 05/19] euscan: Adding freecode handler Uses experimental remote-id "freecode" Signed-off-by: volpino --- pym/euscan/handlers/freecode.py | 48 +++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 pym/euscan/handlers/freecode.py diff --git a/pym/euscan/handlers/freecode.py b/pym/euscan/handlers/freecode.py new file mode 100644 index 0000000..ebfe852 --- /dev/null +++ b/pym/euscan/handlers/freecode.py @@ -0,0 +1,48 @@ +import urllib +import re + +import portage + +from euscan import mangling, helpers, output + +HANDLER_NAME = "freecode" +CONFIDENCE = 100 +PRIORITY = 90 + + +def can_handle(pkg, url=None): + return False + + +def scan_pkg(pkg, options): + cp, ver, rev = portage.pkgsplit(pkg.cpv) + + package = options['data'] + + output.einfo("Using FreeCode handler: " + package) + + fp = urllib.urlopen("http://freecode.com/projects/%s/releases" % package) + content = fp.read() + + result = re.findall( + r'([^<]+)' % package, + content + ) + + ret = [] + for release_id, up_pv in result: + pv = mangling.mangle_version(up_pv, options) + if helpers.version_filtered(cp, ver, pv): + continue + fp = urllib.urlopen("http://freecode.com/projects/%s/releases/%s" % + (package, release_id)) + content = fp.read() + download_page = re.findall(r' Date: Fri, 14 Sep 2012 16:56:52 +0200 Subject: [PATCH 06/19] euscan: Adding naive deb handler, TODO updated Signed-off-by: volpino --- TODO | 18 ++++++++----- pym/euscan/handlers/deb.py | 46 +++++++++++++++++++++++++++++++++ pym/euscan/handlers/freecode.py | 2 +- 3 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 pym/euscan/handlers/deb.py diff --git a/TODO b/TODO index 1cc313f..da0f38f 100644 --- a/TODO +++ b/TODO @@ -41,12 +41,18 @@ packages: ### handlers -- freecode: to scan freecode.com -- remote-id: type deb repository. Example I always scan: - http://dl.google.com/linux/musicmanager/deb/dists/stable/main/binary-i386/Packages - to see if google has release an update, same would help for all google - packages and some of the actual debian packages. - Something similar could be done for rpm repositories. +- remote-id type deb repository: + - Fix handler to support Packages.gz or Packages.bz2 + - find out how to get download url + +### remote-id +- Propose new remote-id: deb + e.g.: + http://mysite.com/deb/dists/stable/main/binary-i386/Packages + +- Propose new remote-id: freecode + e.g.: projectname + euscanwww --------- diff --git a/pym/euscan/handlers/deb.py b/pym/euscan/handlers/deb.py new file mode 100644 index 0000000..156158b --- /dev/null +++ b/pym/euscan/handlers/deb.py @@ -0,0 +1,46 @@ +import urllib +import re + +import portage + +from euscan import mangling, helpers, output + +HANDLER_NAME = "deb" +CONFIDENCE = 100 +PRIORITY = 90 + + +def can_handle(pkg, url=None): + return False + + +def scan_pkg(pkg, options): + cp, ver, rev = portage.pkgsplit(pkg.cpv) + + packages_url, package_name = options['data'].strip().split(" ", 1) + + output.einfo("Using Debian Packages: " + packages_url) + + fp = urllib.urlopen(packages_url) + content = fp.read() + + # TODO: Add support for .gz and .bz2 Packages file + + content = content.split("\n\n") + + result = [] + + for package_info in content: + for line in package_info.split("\n"): + res = re.search("^Version: (.*)$", line) + if res: + result.append(res.group(1)) + + ret = [] + for up_pv in result: + url = "" # TODO: How to find the url? + pv = mangling.mangle_version(up_pv, options) + if helpers.version_filtered(cp, ver, pv): + continue + ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) + return ret diff --git a/pym/euscan/handlers/freecode.py b/pym/euscan/handlers/freecode.py index ebfe852..b811e32 100644 --- a/pym/euscan/handlers/freecode.py +++ b/pym/euscan/handlers/freecode.py @@ -17,7 +17,7 @@ def can_handle(pkg, url=None): def scan_pkg(pkg, options): cp, ver, rev = portage.pkgsplit(pkg.cpv) - package = options['data'] + package = options['data'].strip() output.einfo("Using FreeCode handler: " + package) From 75b5bcc32f781d7ae74af9b272ecc57dce285020 Mon Sep 17 00:00:00 2001 From: volpino Date: Fri, 14 Sep 2012 17:22:18 +0200 Subject: [PATCH 07/19] euscanwww: Adding an option to run upstream scanning after an update in portage Signed-off-by: volpino --- TODO | 5 +---- .../management/commands/scan_portage.py | 6 ++++++ .../djeuscan/processing/scan/scan_portage.py | 21 +++++++++++++++---- euscanwww/djeuscan/tasks.py | 3 ++- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/TODO b/TODO index da0f38f..e93e468 100644 --- a/TODO +++ b/TODO @@ -34,7 +34,7 @@ euscan - Convert subtree - Commit metadata.xml changes to tree -packages: +### packages: - MySQL: should use http://downloads.mysql.com/archives/ - mariadb: should use http://downloads.askmonty.org/MariaDB/+releases/ @@ -101,9 +101,6 @@ The scan process should be faster than the one using euscan. gentoo-packages hav ### Scanning - Remove alive field -- Find a way to rescan upstream of packages recently changed in portage: - - make scan_portage/scan_upstream return a list of packages that were updated (or list of versions ?) - - use that list in tasks.py as scan_upstream (should be easy using chains and subtasks) ### API (LOW-PRIORITY) diff --git a/euscanwww/djeuscan/management/commands/scan_portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py index e40778c..e70affa 100644 --- a/euscanwww/djeuscan/management/commands/scan_portage.py +++ b/euscanwww/djeuscan/management/commands/scan_portage.py @@ -34,6 +34,11 @@ class Command(BaseCommand): dest='purge-versions', default=False, help='Purge old versions'), + make_option('--upstream', + action='store_true', + dest='upstream', + default=False, + help='Scan upstream for updated packages in Portage'), make_option('--no-log', action='store_true', dest='no-log', @@ -66,5 +71,6 @@ class Command(BaseCommand): purge_packages=options["purge-packages"], purge_versions=options["purge-versions"], prefetch=options["prefetch"], + upstream=options["upstream"], logger=logger, ) diff --git a/euscanwww/djeuscan/processing/scan/scan_portage.py b/euscanwww/djeuscan/processing/scan/scan_portage.py index fb450c2..1ce7197 100644 --- a/euscanwww/djeuscan/processing/scan/scan_portage.py +++ b/euscanwww/djeuscan/processing/scan/scan_portage.py @@ -13,16 +13,18 @@ from django.core.management.color import color_style from euscan.version import get_version_type from djeuscan.processing import FakeLogger +from djeuscan.processing.scan.scan_upstream import scan_upstream from djeuscan.models import Package, Version, VersionLog, Category, Overlay class ScanPortage(object): def __init__(self, logger=None, no_log=False, purge_packages=False, - purge_versions=False): + purge_versions=False, upstream=False): self.logger = logger or FakeLogger() self.no_log = no_log self.purge_packages = purge_packages self.purge_versions = purge_versions + self.upstream = upstream self.style = color_style() @@ -196,8 +198,15 @@ class ScanPortage(object): cat, pkg, data['homepage'], data['description'] ) packages_alive.add("%s/%s" % (cat, pkg)) + new_version = False for cpv, slot, overlay, overlay_path in data['versions']: - self.store_version(package, cpv, slot, overlay, overlay_path) + new_version = new_version or self.store_version( + package, cpv, slot, overlay, overlay_path + ) + + # If the package has at least one new version scan upstream for it + if new_version and self.upstream: + scan_upstream([package], self.purge_versions, self.logger) self.purge_old_packages(current_packages, packages_alive) self.purge_old_versions() @@ -260,8 +269,9 @@ class ScanPortage(object): # nothing to do (note: it can't be an upstream version because # overlay can't be empty here) if not created: - return + return False + # New version created self.logger.info('+ [v] %s' % (obj)) if overlay == 'gentoo': @@ -282,6 +292,8 @@ class ScanPortage(object): vtype=obj.vtype, ) + return True + def purge_old_packages(self, packages, alive): if not self.purge_packages: return @@ -324,7 +336,7 @@ class ScanPortage(object): @commit_on_success -def scan_portage(packages=None, category=None, no_log=False, +def scan_portage(packages=None, category=None, no_log=False, upstream=False, purge_packages=False, purge_versions=False, prefetch=False, logger=None): @@ -338,6 +350,7 @@ def scan_portage(packages=None, category=None, no_log=False, no_log=no_log, purge_packages=purge_packages, purge_versions=purge_versions, + upstream=upstream ) logger.info('Scanning portage tree...') diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py index 5e931d2..9c93989 100644 --- a/euscanwww/djeuscan/tasks.py +++ b/euscanwww/djeuscan/tasks.py @@ -113,7 +113,7 @@ def scan_metadata(packages=[], category=None, populate=False): @task -def scan_portage(packages=[], category=None, +def scan_portage(packages=[], category=None, upstream=True, no_log=False, purge_packages=False, purge_versions=False, prefetch=False): """ @@ -133,6 +133,7 @@ def scan_portage(packages=[], category=None, scan.scan_portage( packages=packages, category=category, + upstream=upstream, no_log=no_log, purge_packages=purge_packages, purge_versions=purge_versions, From 3d058946b6825dfe8a7fb86d105890cada6dc771 Mon Sep 17 00:00:00 2001 From: volpino Date: Fri, 21 Sep 2012 17:10:51 +0200 Subject: [PATCH 08/19] euscanwww: making get parameter shorter Signed-off-by: volpino --- euscanwww/djeuscan/feeds.py | 2 +- euscanwww/djeuscan/templates/euscan/world_scan.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/euscanwww/djeuscan/feeds.py b/euscanwww/djeuscan/feeds.py index 3455de5..7399618 100644 --- a/euscanwww/djeuscan/feeds.py +++ b/euscanwww/djeuscan/feeds.py @@ -247,7 +247,7 @@ class WorldScanFeed(BaseFeed): def _items(self, data): packages = [] - for pkg_name in data["options"].getlist("package", []): + for pkg_name in data["options"].getlist("p", []): if "/" in pkg_name: cat, name = pkg_name.split("/", 1) pkg = Package.objects.filter(name=name, category=cat) diff --git a/euscanwww/djeuscan/templates/euscan/world_scan.html b/euscanwww/djeuscan/templates/euscan/world_scan.html index 30a029e..14dd428 100644 --- a/euscanwww/djeuscan/templates/euscan/world_scan.html +++ b/euscanwww/djeuscan/templates/euscan/world_scan.html @@ -12,7 +12,7 @@
  • feed + href="{% url "world_scan_feed" %}?p={{ packages|join:"&p=" }}"> World scan feed
    • From ef201397308481672674cc86658e5efe9d51c1b3 Mon Sep 17 00:00:00 2001 From: volpino Date: Fri, 21 Sep 2012 17:12:43 +0200 Subject: [PATCH 09/19] euscan: Removing http cache from TODO Signed-off-by: volpino --- TODO | 4 ---- 1 file changed, 4 deletions(-) diff --git a/TODO b/TODO index e93e468..994f72b 100644 --- a/TODO +++ b/TODO @@ -14,10 +14,6 @@ euscan - html output -### Misc - -- Add a HTTP cache (configurable) - ### Tests - Write tests for handlers From 565c5823c1cc35b5c447468d1aa9beecbd0d0e9f Mon Sep 17 00:00:00 2001 From: volpino Date: Fri, 21 Sep 2012 19:03:53 +0200 Subject: [PATCH 10/19] euscan: deb handler - adding support for Packages.gz or Packages.bz2 and version detection fix Signed-off-by: volpino --- TODO | 3 +-- pym/euscan/handlers/deb.py | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/TODO b/TODO index 994f72b..9fda0b9 100644 --- a/TODO +++ b/TODO @@ -38,8 +38,7 @@ euscan ### handlers - remote-id type deb repository: - - Fix handler to support Packages.gz or Packages.bz2 - - find out how to get download url + - find out how to get download url (not sure it's possible) ### remote-id - Propose new remote-id: deb diff --git a/pym/euscan/handlers/deb.py b/pym/euscan/handlers/deb.py index 156158b..b559263 100644 --- a/pym/euscan/handlers/deb.py +++ b/pym/euscan/handlers/deb.py @@ -1,5 +1,7 @@ import urllib import re +import bz2 +import zlib import portage @@ -24,17 +26,22 @@ def scan_pkg(pkg, options): fp = urllib.urlopen(packages_url) content = fp.read() - # TODO: Add support for .gz and .bz2 Packages file + # Support for .gz and .bz2 Packages file + if packages_url.endswith(".bz2"): + content = bz2.decompress(content) + if packages_url.endswith(".gz"): + content = zlib.decompress(content, 16 + zlib.MAX_WBITS) content = content.split("\n\n") result = [] for package_info in content: - for line in package_info.split("\n"): - res = re.search("^Version: (.*)$", line) - if res: - result.append(res.group(1)) + package_line = re.search(r"^Package: (.*)$", package_info, re.M) + version_line = re.search(r"^Version: (.*)$", package_info, re.M) + if package_line and package_line.group(1) == package_name: + if version_line: + result.append(version_line.group(1)) ret = [] for up_pv in result: From ddd8be39f5ac4c7a9f3997473bd239088b1403ae Mon Sep 17 00:00:00 2001 From: volpino Date: Sat, 22 Sep 2012 15:45:07 +0200 Subject: [PATCH 11/19] euscanwww: Wrapping herds and maintainers in

    to put them in newlines Signed-off-by: volpino --- .../templates/euscan/_package_details.html | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/euscanwww/djeuscan/templates/euscan/_package_details.html b/euscanwww/djeuscan/templates/euscan/_package_details.html index b6a85ec..8746088 100644 --- a/euscanwww/djeuscan/templates/euscan/_package_details.html +++ b/euscanwww/djeuscan/templates/euscan/_package_details.html @@ -18,10 +18,12 @@

    Herds
    {% for herd in package.herds.all %} - - {{ herd.herd }} - - <{{ herd.email }}> +

    + + {{ herd.herd }} + + <{{ herd.email }}> +

    {% endfor %}
    {% endif %} @@ -29,16 +31,18 @@
    Maintainers
    {% for maintainer in package.maintainers.all %} - {% if maintainer.name != maintainer.email %} - - {{ maintainer.name }} - - <{{ maintainer.email }}> - {% else %} - - {{ maintainer.email }} - - {% endif %} +

    + {% if maintainer.name != maintainer.email %} + + {{ maintainer.name }} + + <{{ maintainer.email }}> + {% else %} + + {{ maintainer.email }} + + {% endif %} +

    {% endfor %}
    {% endif %} From c519f6b28a4becb227b74556cbbaed99bcef0553 Mon Sep 17 00:00:00 2001 From: volpino Date: Sat, 22 Sep 2012 16:08:38 +0200 Subject: [PATCH 12/19] euscan: last_version property added Signed-off-by: volpino --- TODO | 2 -- euscanwww/djeuscan/models.py | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/TODO b/TODO index 9fda0b9..a0e3a26 100644 --- a/TODO +++ b/TODO @@ -79,8 +79,6 @@ Move log models into djeuscanhistory ? - Url ad ManyToMany - Arches and Keyword - Metadata, herds, maintainers and homepage are per-version, not per package. Store it in Version instead. - Define last_version that refers to the last version (can be last_version_gentoo or last_version_overlay) - Fix update_counter, to update this field. ### djportage (LOW-PRIORITY)) diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py index cb0e0e0..a328522 100644 --- a/euscanwww/djeuscan/models.py +++ b/euscanwww/djeuscan/models.py @@ -116,6 +116,20 @@ class Package(models.Model): def homepages(self): return self.homepage.split(' ') + @property + def last_version(self): + from euscan.helpers import vercmp + + versions = [ + self.last_version_gentoo, + self.last_version_overlay, + self.last_version_upstream + ] + _cmp = lambda x, y: vercmp( + unicode(self), x.version if x else "", y.version if y else "" + ) + return sorted(versions, cmp=_cmp)[-1] + class Version(models.Model): """ From 4b3e0bd525b150e3816099051e06ef44101223d3 Mon Sep 17 00:00:00 2001 From: volpino Date: Mon, 24 Sep 2012 15:36:11 +0200 Subject: [PATCH 13/19] euscanwww: Upgrading Bootstrap toogle buttons Signed-off-by: volpino --- euscanwww/htdocs/css/toggle_buttons.css | 166 ++++++------ euscanwww/htdocs/js/jquery.toggle.buttons.js | 263 ++++++++++++++----- 2 files changed, 282 insertions(+), 147 deletions(-) diff --git a/euscanwww/htdocs/css/toggle_buttons.css b/euscanwww/htdocs/css/toggle_buttons.css index 29a5289..c884ba4 100644 --- a/euscanwww/htdocs/css/toggle_buttons.css +++ b/euscanwww/htdocs/css/toggle_buttons.css @@ -1,25 +1,17 @@ -/* line 7, ../sass/screen.scss */ +/* line 11, ../sass/bootstrap-toggle-buttons.scss */ .toggle-button { display: inline-block; cursor: pointer; - background: #0088CC; - background-image: -webkit-gradient(linear, 50% 0%, 50% 100%, color-stop(0%, #0088cc), color-stop(100%, #0055cc)); - background-image: -webkit-linear-gradient(top, #0088cc, #0055cc); - background-image: -moz-linear-gradient(top, #0088cc, #0055cc); - background-image: -o-linear-gradient(top, #0088cc, #0055cc); - background-image: linear-gradient(top, #0088cc, #0055cc); - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - -ms-border-radius: 4px; - -o-border-radius: 4px; - border-radius: 4px; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + -ms-border-radius: 5px; + -o-border-radius: 5px; + border-radius: 5px; border: 1px solid; border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); position: relative; - bottom: -2px; text-align: left; - min-height: 25px; - max-height: 25px; + overflow: hidden; -webkit-touch-callout: none; -webkit-user-select: none; -khtml-user-select: none; @@ -27,50 +19,46 @@ -ms-user-select: none; user-select: none; } -/* line 29, ../sass/screen.scss */ -.toggle-button:before, .toggle-button:after { - line-height: 25px; - font-weight: bold; - letter-spacing: .4px; +/* line 29, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button.deactivate { + filter: progid:DXImageTransform.Microsoft.Alpha(Opacity=50); + opacity: 0.5; + cursor: default !important; } -/* line 35, ../sass/screen.scss */ -.toggle-button:before { - color: #fefefe; - padding-left: 0%; - margin-left: 10px; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.3); - content: attr(data-enabled); +/* line 32, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button.deactivate label, .toggle-button.deactivate span { + cursor: default !important; } -/* line 43, ../sass/screen.scss */ -.toggle-button.disabled { - background: #fefefe; - background-image: -webkit-gradient(linear, 50% 0%, 50% 100%, color-stop(0%, #fefefe), color-stop(100%, #e6e6e6)); - background-image: -webkit-linear-gradient(top, #fefefe, #e6e6e6); - background-image: -moz-linear-gradient(top, #fefefe, #e6e6e6); - background-image: -o-linear-gradient(top, #fefefe, #e6e6e6); - background-image: linear-gradient(top, #fefefe, #e6e6e6); - border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); -} -/* line 47, ../sass/screen.scss */ -.toggle-button.disabled:before { - color: #555555; - padding-left: 50%; - margin-left: 10px; - text-shadow: 0 1px 0 rgba(255, 255, 255, 0.3); - content: attr(data-disabled); -} -/* line 54, ../sass/screen.scss */ -.toggle-button.disabled label { - margin-left: -1px; - left: 0%; -} -/* line 60, ../sass/screen.scss */ -.toggle-button label { - cursor: pointer; +/* line 36, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button > div { + display: inline-block; + width: 150px; position: absolute; - width: 50%; - height: 25px; + top: 0; +} +/* line 41, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button > div.disabled { + left: -50%; +} +/* line 45, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button input[type=checkbox] { + display: none; +} +/* line 53, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button span, .toggle-button label { + cursor: pointer; + position: relative; + float: left; + display: inline-block; +} +/* line 60, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button label { background: #fefefe; + margin-left: -4px; + margin-right: -4px; + border: 1px solid #E6E6E6; + margin-top: -1px; + z-index: 100; background-image: -webkit-gradient(linear, 50% 0%, 50% 100%, color-stop(0%, #fefefe), color-stop(100%, #e6e6e6)); background-image: -webkit-linear-gradient(top, #fefefe, #e6e6e6); background-image: -moz-linear-gradient(top, #fefefe, #e6e6e6); @@ -81,23 +69,43 @@ -ms-border-radius: 4px; -o-border-radius: 4px; border-radius: 4px; - border: 1px solid; - border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); - top: -1px; - margin-left: 0; - left: 50%; } -/* line 75, ../sass/screen.scss */ -.toggle-button input[type=checkbox] { - display: none; +/* line 72, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button span { + color: #fefefe; + text-align: center; + font-weight: bold; + z-index: 1; } -/* line 84, ../sass/screen.scss */ -.toggle-button.primary:before, .toggle-button.info:before, .toggle-button.success:before, .toggle-button.warning:before, .toggle-button.danger:before { - color: white; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.3); +/* line 78, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button span.labelLeft { + -moz-border-radius-topleft: 4px; + -webkit-border-top-left-radius: 4px; + border-top-left-radius: 4px; + -moz-border-radius-bottomleft: 4px; + -webkit-border-bottom-left-radius: 4px; + border-bottom-left-radius: 4px; + padding-left: 3px; } -/* line 90, ../sass/screen.scss */ -.toggle-button.primary { +/* line 83, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button span.labelRight { + -moz-border-radius-topright: 4px; + -webkit-border-top-right-radius: 4px; + border-top-right-radius: 4px; + -moz-border-radius-bottomright: 4px; + -webkit-border-bottom-right-radius: 4px; + border-bottom-right-radius: 4px; + color: black; + background-image: -webkit-gradient(linear, 50% 0%, 50% 100%, color-stop(0%, #fefefe), color-stop(100%, #e6e6e6)); + background-image: -webkit-linear-gradient(top, #fefefe, #e6e6e6); + background-image: -moz-linear-gradient(top, #fefefe, #e6e6e6); + background-image: -o-linear-gradient(top, #fefefe, #e6e6e6); + background-image: linear-gradient(top, #fefefe, #e6e6e6); + padding-right: 3px; +} +/* line 91, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button span.primary, .toggle-button span.labelLeft { + color: #fefefe; background: #0088cc; background-image: -webkit-gradient(linear, 50% 0%, 50% 100%, color-stop(0%, #0088cc), color-stop(100%, #0055cc)); background-image: -webkit-linear-gradient(top, #0088cc, #0055cc); @@ -105,8 +113,9 @@ background-image: -o-linear-gradient(top, #0088cc, #0055cc); background-image: linear-gradient(top, #0088cc, #0055cc); } -/* line 94, ../sass/screen.scss */ -.toggle-button.info { +/* line 96, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button span.info { + color: #fefefe; background: #5bc0de; background-image: -webkit-gradient(linear, 50% 0%, 50% 100%, color-stop(0%, #5bc0de), color-stop(100%, #2f96b4)); background-image: -webkit-linear-gradient(top, #5bc0de, #2f96b4); @@ -114,8 +123,9 @@ background-image: -o-linear-gradient(top, #5bc0de, #2f96b4); background-image: linear-gradient(top, #5bc0de, #2f96b4); } -/* line 99, ../sass/screen.scss */ -.toggle-button.success { +/* line 102, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button span.success { + color: #fefefe; background: #62c462; background-image: -webkit-gradient(linear, 50% 0%, 50% 100%, color-stop(0%, #62c462), color-stop(100%, #51a351)); background-image: -webkit-linear-gradient(top, #62c462, #51a351); @@ -123,8 +133,9 @@ background-image: -o-linear-gradient(top, #62c462, #51a351); background-image: linear-gradient(top, #62c462, #51a351); } -/* line 104, ../sass/screen.scss */ -.toggle-button.warning { +/* line 108, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button span.warning { + color: #fefefe; background: #dbb450; background-image: -webkit-gradient(linear, 50% 0%, 50% 100%, color-stop(0%, #dbb450), color-stop(100%, #f89406)); background-image: -webkit-linear-gradient(top, #dbb450, #f89406); @@ -132,8 +143,9 @@ background-image: -o-linear-gradient(top, #dbb450, #f89406); background-image: linear-gradient(top, #dbb450, #f89406); } -/* line 109, ../sass/screen.scss */ -.toggle-button.danger { +/* line 114, ../sass/bootstrap-toggle-buttons.scss */ +.toggle-button span.danger { + color: #fefefe; background: #ee5f5b; background-image: -webkit-gradient(linear, 50% 0%, 50% 100%, color-stop(0%, #ee5f5b), color-stop(100%, #bd362f)); background-image: -webkit-linear-gradient(top, #ee5f5b, #bd362f); diff --git a/euscanwww/htdocs/js/jquery.toggle.buttons.js b/euscanwww/htdocs/js/jquery.toggle.buttons.js index bea6465..363850e 100644 --- a/euscanwww/htdocs/js/jquery.toggle.buttons.js +++ b/euscanwww/htdocs/js/jquery.toggle.buttons.js @@ -1,96 +1,211 @@ !function ($) { "use strict"; - // version: 1.3 + // version: 2.2 + // by Mattia Larentis - follow me on twitter! @SpiritualGuru - $.fn.toggleButtons = function (opt) { + $.fn.toggleButtons = function (method) { var $element - , $labelEnabled - , options - , active - , styleActive - , styleDisabled - , animationCss + , $div , transitionSpeed = 0.05 - , defaultSpeed = 0.05; + , methods = { + init: function (opt) { + this.each(function () { + var $spanLeft + , $spanRight + , options + , moving; - this.each(function () { - $element = $(this); + $element = $(this); + $element.addClass('toggle-button'); - options = $.extend({}, $.fn.toggleButtons.defaults, opt); + options = $.extend({}, $.fn.toggleButtons.defaults, opt); + $(this).data('options', options); - $element.addClass('toggle-button'); + $spanLeft = $('').addClass("labelLeft").text(options.label.enabled === undefined ? "ON" : options.label.enabled); + $spanRight = $('').addClass("labelRight").text(options.label.disabled === undefined ? "OFF " : options.label.disabled); - $labelEnabled = $('').attr('for', $element.find('input').attr('id')); - $element.append($labelEnabled); + // html layout + $div = $element.find('input').wrap($('
    ')).parent(); + $div.append($spanLeft); + $div.append($('').attr('for', $element.find('input').attr('id'))); + $div.append($spanRight); - if (options.animated) { - $element.addClass('toggle-button-animated'); + if ($element.find('input').is(':checked')) + $element.find('>div').css('left', "0"); + else $element.find('>div').css('left', "-50%"); - if (options.transitionSpeed !== undefined) - if (/^(\d*%$)/.test(options.transitionSpeed)) // is a percent value? - transitionSpeed = defaultSpeed * parseInt(options.transitionSpeed) / 100; - else - transitionSpeed = options.transitionSpeed; + if (options.animated) { + if (options.transitionSpeed !== undefined) + if (/^(\d*%$)/.test(options.transitionSpeed)) // is a percent value? + transitionSpeed = 0.05 * parseInt(options.transitionSpeed) / 100; + else + transitionSpeed = options.transitionSpeed; + } + else transitionSpeed = 0; - animationCss = ["-webkit-", "-moz-", "-o-", ""]; - $(animationCss).each(function () { - $element.find('label').css(this + 'transition', 'all ' + transitionSpeed + 's'); - }); - } - - $element.css('width', options.width); - - $element.attr("data-enabled", options.label.enabled === undefined ? "ON" : options.label.enabled); - $element.attr("data-disabled", options.label.disabled === undefined ? "OFF " : options.label.disabled); - - active = $element.find('input').is(':checked'); - - if (!active) - $element.addClass('disabled'); - - styleActive = options.style.enabled === undefined ? "" : options.style.enabled; - styleDisabled = options.style.disabled === undefined ? "" : options.style.disabled; - - if (active && styleActive !== undefined) - $element.addClass(styleActive); - if (!active && styleDisabled !== undefined) - $element.addClass(styleDisabled); - - $element.on('click', function (e) { - if ($(e.target).is('input')) - return true; - - e.stopPropagation(); - $(this).find('label').click(); - }); + $(this).data('transitionSpeed', transitionSpeed * 1000); - $element.find('label').on('click', function (e) { - e.stopPropagation(); - e.preventDefault(); + options["width"] /= 2; - $element = $(this).parent(); + // width of the bootstrap-toggle-button + $element + .css('width', options.width * 2) + .find('>div').css('width', options.width * 3) + .find('>span, >label').css('width', options.width); - $element - .delay(transitionSpeed * 500).queue(function () { - $(this).toggleClass('disabled') - .toggleClass(styleActive) - .toggleClass(styleDisabled) - .dequeue(); - }); + // height of the bootstrap-toggle-button + $element + .css('height', options.height) + .find('span, label') + .css('height', options.height) + .filter('span') + .css('line-height', options.height + "px"); - active = !($element.find('input').is(':checked')); + if ($element.find('input').is(':disabled')) + $(this).addClass('deactivate'); - $element.find('input').attr('checked', active); - options.onChange($element, active, e); - }); - }); + $element.find('span').css(options.font); + + + // enabled custom color + if (options.style.enabled === undefined) { + if (options.style.custom !== undefined && options.style.custom.enabled !== undefined) { + $spanLeft.css('color', options.style.custom.enabledColor); + if (options.style.custom.enabledGradient === undefined) + $spanLeft.css('background', options.style.custom.enabled); + else $.each(["-webkit-", "-moz-", "-o-", ""], function (i, el) { + $spanLeft.css('background-image', el + 'linear-gradient(top, ' + options.style.custom.enabled + ',' + options.style.custom.enabledGradient + ')'); + }); + } + } + else $spanLeft.addClass(options.style.enabled); + + // disabled custom color + if (options.style.disabled === undefined) { + if (options.style.custom !== undefined && options.style.custom.disabled !== undefined) { + $spanRight.css('color', options.style.custom.disabledColor); + if (options.style.custom.disabledGradient === undefined) + $spanRight.css('background', options.style.custom.disabled); + else $.each(["-webkit-", "-moz-", "-o-", ""], function (i, el) { + $spanRight.css('background-image', el + 'linear-gradient(top, ' + options.style.custom.disabled + ',' + options.style.custom.disabledGradient + ')'); + }); + } + } + else $spanRight.addClass(options.style.disabled); + + + var changeStatus = function ($this) { + $this.siblings('label').trigger('mousedown').trigger('mouseup').trigger('click'); + }; + + $spanLeft.on('click', function (e) { + changeStatus($(this)); + }); + $spanRight.on('click', function (e) { + changeStatus($(this)); + }); + + $('.toggle-button').find('input').on('change', function (e) { + var $element = $(this).parent() + , active = $(this).is(':checked') + , $toggleButton = $(this).closest('.toggle-button'); + + e.preventDefault(); + e.stopImmediatePropagation(); + + $element.animate({'left': active ? '0' : '-50%'}, $toggleButton.data('transitionSpeed')); + + options = $toggleButton.data('options'); + options.onChange($element, active, e); + }); + + $('.toggle-button').find('label').on('mousedown', function (e) { + moving = false; + + e.preventDefault(); + e.stopImmediatePropagation(); + + if ($(this).closest('.toggle-button').is('.deactivate')) + $(this).unbind('click'); + else { + $(this).on('mousemove', function (e) { + var $element = $(this).closest('.toggle-button') + , relativeX = e.pageX - $element.offset().left + , percent = ((relativeX / (options.width * 2)) * 100); + moving = true; + + if (percent < 25) + percent = 25; + else if (percent > 75) + percent = 75; + + $element.find('>div').css('left', (percent - 75) + "%") + }); + + $(this).on('click', function (e) { + var $target = $(e.target) + , $myCheckBox = $target.siblings('input'); + + e.stopImmediatePropagation(); + e.preventDefault(); + $(this).unbind('mouseleave'); + + if (moving) + if (parseInt($(this).parent().css('left')) < -25) + $myCheckBox.attr('checked', false); + else $myCheckBox.attr('checked', true); + else $myCheckBox.attr("checked", !$myCheckBox.is(":checked")); + + $myCheckBox.trigger('change'); + }); + + $(this).on('mouseleave', function (e) { + var $myCheckBox = $(this).siblings('input'); + + e.preventDefault(); + e.stopImmediatePropagation(); + + $(this).unbind('mouseleave'); + $(this).trigger('mouseup'); + + if (parseInt($(this).parent().css('left')) < -25) + $myCheckBox.attr('checked', false); + else $myCheckBox.attr('checked', true); + + $myCheckBox.trigger('change'); + }); + + $(this).on('mouseup', function (e) { + e.stopImmediatePropagation(); + e.preventDefault(); + $(this).unbind('mousemove'); + }); + } + }); + } + ) + ; + }, + toggleActivation: function () { + $(this).toggleClass('deactivate'); + } + }; + + if (methods[method]) + return methods[method].apply(this, Array.prototype.slice.call(arguments, 1)); + else if (typeof method === 'object' || !method) + return methods.init.apply(this, arguments); + else + $.error('Method ' + method + ' does not exist on jQuery.tooltip'); }; $.fn.toggleButtons.defaults = { onChange: function () { }, width: 100, + height: 25, + font: {}, animated: true, transitionSpeed: undefined, label: { @@ -99,7 +214,15 @@ }, style: { enabled: undefined, - disabled: undefined + disabled: undefined, + custom: { + enabled: undefined, + enabledGradient: undefined, + enabledColor: "#FFFFFF", + disabled: undefined, + disabledGradient: undefined, + disabledColor: "#FFFFFF" + } } }; }($); From 975f7b1dbf34175531b8580f688cbb7c2d65895f Mon Sep 17 00:00:00 2001 From: volpino Date: Mon, 24 Sep 2012 16:47:21 +0200 Subject: [PATCH 14/19] euscanwww: Adding an about config page Signed-off-by: volpino --- TODO | 1 - euscanwww/djeuscan/templates/_base.html | 1 + .../djeuscan/templates/euscan/config.html | 95 +++++++++++++++++++ euscanwww/djeuscan/urls.py | 1 + euscanwww/djeuscan/views.py | 16 ++++ euscanwww/htdocs/css/style.css | 8 -- 6 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 euscanwww/djeuscan/templates/euscan/config.html diff --git a/TODO b/TODO index a0e3a26..932c479 100644 --- a/TODO +++ b/TODO @@ -52,7 +52,6 @@ euscan euscanwww --------- -- Add an /about/config page that describe the current config (overlays, stuff in make.conf, euscan default settings, etc..) - Always keep in db all found versions (when using an API only?). But don't display them if older than current packaged version, except maybe in the "upstream_version" column. ### packages diff --git a/euscanwww/djeuscan/templates/_base.html b/euscanwww/djeuscan/templates/_base.html index 2d26a33..954e350 100644 --- a/euscanwww/djeuscan/templates/_base.html +++ b/euscanwww/djeuscan/templates/_base.html @@ -86,6 +86,7 @@
  • ---
  • API
  • Feeds
    • +
  • Server config
  • About
    • {% endblock %} diff --git a/euscanwww/djeuscan/templates/euscan/config.html b/euscanwww/djeuscan/templates/euscan/config.html new file mode 100644 index 0000000..9bbaf14 --- /dev/null +++ b/euscanwww/djeuscan/templates/euscan/config.html @@ -0,0 +1,95 @@ +{% extends "_base.html" %} + +{% block content %} +

    Server configuration

    + + + + {% for key, value in CONFIG.items %} + + + + + {% endfor %} +
    +

    Euscan config

    +
    {{ key }}{{ value }}
    + +
    + + + + {% for item in BLACKLIST_VERSIONS %} + + + + {% endfor %} +
    +

    Versions blacklist

    +
    {{ item }}
    + +
    + + + + {% for item in BLACKLIST_PACKAGES %} + + + + {% endfor %} +
    +

    Packages blacklist

    +
    {{ item }}
    + +
    + + + + {% for item in SCANDIR_BLACKLIST_URLS %} + + + + {% endfor %} +
    +

    Scandir blacklist

    +
    {{ item }}
    + +
    + + + + {% for item in BRUTEFORCE_BLACKLIST_PACKAGES %} + + + + {% endfor %} +
    +

    Bruteforce packages blacklist

    +
    {{ item }}
    + +
    + + + + {% for item in BRUTEFORCE_BLACKLIST_URLS %} + + + + {% endfor %} +
    +

    Bruteforce urls blacklist

    +
    {{ item }}
    + +
    + + + + {% for item in ROBOTS_TXT_BLACKLIST_DOMAINS %} + + + + {% endfor %} +
    +

    Robots.txt domain blacklist

    +
    {{ item }}
    +{% endblock %} diff --git a/euscanwww/djeuscan/urls.py b/euscanwww/djeuscan/urls.py index e156a4b..789fc9f 100644 --- a/euscanwww/djeuscan/urls.py +++ b/euscanwww/djeuscan/urls.py @@ -113,6 +113,7 @@ urlpatterns = patterns('djeuscan.views', url(r'^about/$', 'about', name="about"), url(r'^about/api$', 'api', name="api"), url(r'^about/feeds$', 'feeds', name="feeds"), + url(r'^about/config$', 'config', name="config"), url(r'^statistics/$', 'statistics', name="statistics"), url(r'^statistics/charts/(?P[\w\-]+).png$', 'chart', name="chart"), url(r'^world/$', 'world', name="world"), diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py index bb60df2..cc0d2f2 100644 --- a/euscanwww/djeuscan/views.py +++ b/euscanwww/djeuscan/views.py @@ -311,6 +311,22 @@ def feeds(request): return {} +@render_to("euscan/config.html") +def config(request): + from euscan import CONFIG, BLACKLIST_VERSIONS, BLACKLIST_PACKAGES, \ + SCANDIR_BLACKLIST_URLS, BRUTEFORCE_BLACKLIST_PACKAGES, \ + BRUTEFORCE_BLACKLIST_URLS, ROBOTS_TXT_BLACKLIST_DOMAINS + return { + "CONFIG": CONFIG, + "BLACKLIST_VERSIONS": BLACKLIST_VERSIONS, + "BLACKLIST_PACKAGES": BLACKLIST_PACKAGES, + "SCANDIR_BLACKLIST_URLS": SCANDIR_BLACKLIST_URLS, + "BRUTEFORCE_BLACKLIST_PACKAGES": BRUTEFORCE_BLACKLIST_PACKAGES, + "BRUTEFORCE_BLACKLIST_URLS": BRUTEFORCE_BLACKLIST_URLS, + "ROBOTS_TXT_BLACKLIST_DOMAINS": ROBOTS_TXT_BLACKLIST_DOMAINS, + } + + @render_to("euscan/statistics.html") def statistics(request): return {} diff --git a/euscanwww/htdocs/css/style.css b/euscanwww/htdocs/css/style.css index 2877499..bc3cdcb 100644 --- a/euscanwww/htdocs/css/style.css +++ b/euscanwww/htdocs/css/style.css @@ -117,14 +117,6 @@ dd { padding: 5px; } -hr { - margin: 0.3em 1em 0.3em 1em; - height: 1px; - border: #bcbcbc dashed; - border-width: 0 0 1px 0; -} - - .table { width: 60% !important; margin: auto; From 808acfe13aa299cc0d800c9e9c2e8dec764b8be3 Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Mon, 29 Oct 2012 12:46:37 +0100 Subject: [PATCH 15/19] django-recaptcha 0.0.5 compat Signed-off-by: Corentin Chary --- euscanwww/djeuscan/processing/scan/scan_upstream.py | 2 +- euscanwww/euscanwww/captcha/__init__.py | 6 ++++++ euscanwww/euscanwww/captcha/forms.py | 5 +++++ euscanwww/euscanwww/captcha/urls.py | 12 ++++++++++++ euscanwww/euscanwww/settings.py | 1 + euscanwww/euscanwww/urls.py | 2 +- 6 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 euscanwww/euscanwww/captcha/__init__.py create mode 100644 euscanwww/euscanwww/captcha/forms.py create mode 100644 euscanwww/euscanwww/captcha/urls.py diff --git a/euscanwww/djeuscan/processing/scan/scan_upstream.py b/euscanwww/djeuscan/processing/scan/scan_upstream.py index bc9159b..6ec2b76 100644 --- a/euscanwww/djeuscan/processing/scan/scan_upstream.py +++ b/euscanwww/djeuscan/processing/scan/scan_upstream.py @@ -34,7 +34,7 @@ class ScanUpstream(object): "Error while scanning upstream for package %s!", package ) - self.logger.debug( + self.logger.info( "Error %s", out_json ) diff --git a/euscanwww/euscanwww/captcha/__init__.py b/euscanwww/euscanwww/captcha/__init__.py new file mode 100644 index 0000000..a6b282a --- /dev/null +++ b/euscanwww/euscanwww/captcha/__init__.py @@ -0,0 +1,6 @@ +from registration.backends.default import DefaultBackend +from forms import RegistrationFormCaptcha + +class CaptchaDefaultBackend(DefaultBackend): + def get_form_class(self, request): + return RegistrationFormCaptcha diff --git a/euscanwww/euscanwww/captcha/forms.py b/euscanwww/euscanwww/captcha/forms.py new file mode 100644 index 0000000..ca4a876 --- /dev/null +++ b/euscanwww/euscanwww/captcha/forms.py @@ -0,0 +1,5 @@ +from captcha.fields import ReCaptchaField +from registration.forms import RegistrationForm + +class RegistrationFormCaptcha(RegistrationForm): + captcha = ReCaptchaField(attrs={'theme': 'white'}) diff --git a/euscanwww/euscanwww/captcha/urls.py b/euscanwww/euscanwww/captcha/urls.py new file mode 100644 index 0000000..76cbd2e --- /dev/null +++ b/euscanwww/euscanwww/captcha/urls.py @@ -0,0 +1,12 @@ +from django.conf.urls.defaults import patterns, url + +from registration.views import register + +urlpatterns = patterns('', + url( + r'^register/$', + register, + {'backend': 'euscanwww.captcha.CaptchaDefaultBackend'}, + name='registration_register' + ), +) diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py index d6b8616..5705c3b 100644 --- a/euscanwww/euscanwww/settings.py +++ b/euscanwww/euscanwww/settings.py @@ -267,3 +267,4 @@ except ImportError, ex: os.environ['ROOT'] = PORTAGE_ROOT os.environ['PORTAGE_CONFIGROOT'] = PORTAGE_CONFIGROOT os.environ['EIX_CACHEFILE'] = EIX_CACHEFILE +os.environ['http_proxy'] = '127.0.0.1:8123' diff --git a/euscanwww/euscanwww/urls.py b/euscanwww/euscanwww/urls.py index d6090b5..b104339 100644 --- a/euscanwww/euscanwww/urls.py +++ b/euscanwww/euscanwww/urls.py @@ -8,7 +8,7 @@ urlpatterns = patterns('', url(r'^', include('djeuscan.urls')), url(r'^admin/', include(admin.site.urls)), - url(r'^accounts/', include('captcha.backends.default.urls')), + url(r'^accounts/', include('euscanwww.captcha.urls')), url(r'^accounts/', include('registration.backends.default.urls')), ) From 5a8d7e8932fcde1b466a4faba8a98c6d17405bd0 Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Mon, 29 Oct 2012 12:47:32 +0100 Subject: [PATCH 16/19] euscanwww: use maintainers names, not ids Signed-off-by: Corentin Chary --- euscanwww/djeuscan/api/handlers.py | 8 ++++-- euscanwww/djeuscan/api/urls.py | 2 ++ euscanwww/djeuscan/feeds.py | 14 +++++++--- .../templates/euscan/_maintainers_table.html | 6 ++--- .../templates/euscan/_package_details.html | 4 +-- .../templates/euscan/accounts/index.html | 2 +- .../djeuscan/templates/euscan/maintainer.html | 22 ++++++++-------- euscanwww/djeuscan/urls.py | 15 +++++------ euscanwww/djeuscan/views.py | 26 ++++++++++++------- 9 files changed, 57 insertions(+), 42 deletions(-) diff --git a/euscanwww/djeuscan/api/handlers.py b/euscanwww/djeuscan/api/handlers.py index 18e1aca..922fe16 100644 --- a/euscanwww/djeuscan/api/handlers.py +++ b/euscanwww/djeuscan/api/handlers.py @@ -97,8 +97,12 @@ class PackagesHandler(AnonymousBaseHandler): herd = Herd.objects.get(herd=kwargs['herd']) packages = Package.objects.for_herd(herd, last_versions=True) data = {'herd': herd} - elif 'maintainer_id' in kwargs: - maintainer = Maintainer.objects.get(id=kwargs['maintainer_id']) + elif 'maintainer_id' in kwargs or 'maintainer_email' in kwargs: + if 'maintainer_id' in kwargs: + maintainer = Maintainer.objects.get(id=kwargs['maintainer_id']) + elif 'maintainer_email' in kwargs: + maintainer = Maintainer.objects.get(email=kwargs['maintainer_email']) + packages = Package.objects.for_maintainer( maintainer, last_versions=True diff --git a/euscanwww/djeuscan/api/urls.py b/euscanwww/djeuscan/api/urls.py index ffab482..578a8be 100644 --- a/euscanwww/djeuscan/api/urls.py +++ b/euscanwww/djeuscan/api/urls.py @@ -24,6 +24,8 @@ urlpatterns = patterns('djeuscan.api.views', url(r'^1.0/packages/by-maintainer/(?P\d+)\.(?P.+)$', packages_handler, name='api.views.packages'), + url(r'^1.0/packages/by-maintainer/(?P[^/]+)\.(?P.+)$', + packages_handler, name='api.views.packages'), url(r'^1.0/packages/by-herd/(?P[\@\{\}\w+.-]*)\.(?P.+)?$', packages_handler, name='api.views.packages'), url(r'^1.0/packages/by-category/(?P[\w+][\w+.-]*)\.(?P.+)?$', diff --git a/euscanwww/djeuscan/feeds.py b/euscanwww/djeuscan/feeds.py index 7399618..98590a9 100644 --- a/euscanwww/djeuscan/feeds.py +++ b/euscanwww/djeuscan/feeds.py @@ -146,9 +146,16 @@ class PackageFeed(BaseFeed): class MaintainerFeed(BaseFeed): feed_type = Atom1Feed - def get_object(self, request, maintainer_id): + def get_object(self, request, maintainer_id=None, maintainer_email=None): + print maintainer_id, maintainer_email + if maintainer_id: + obj = get_object_or_404(Maintainer, id=maintainer_id) + else: + obj = get_object_or_404(Maintainer, email=maintainer_email) + + print obj return { - "obj": get_object_or_404(Maintainer, id=maintainer_id), + "obj": obj, "options": request.GET, } @@ -159,8 +166,7 @@ class MaintainerFeed(BaseFeed): return "Last changes for %s" % data["obj"] def link(self, data): - return reverse('djeuscan.views.maintainer', - kwargs={'maintainer_id': data["obj"].id}) + return reverse('djeuscan.views.maintainer', args=[data["obj"].email]) def _items(self, data): return VersionLog.objects.for_maintainer(data["obj"]), 50 diff --git a/euscanwww/djeuscan/templates/euscan/_maintainers_table.html b/euscanwww/djeuscan/templates/euscan/_maintainers_table.html index bfbdd21..665e3e8 100644 --- a/euscanwww/djeuscan/templates/euscan/_maintainers_table.html +++ b/euscanwww/djeuscan/templates/euscan/_maintainers_table.html @@ -18,7 +18,7 @@ {% for maintainer in maintainers %} - + {% if maintainer.maintainers__name != maintainer.maintainers__email %} {{ maintainer.maintainers__name }} <{{ maintainer.maintainers__email }}> {% else %} @@ -31,8 +31,8 @@ {% if extras %} - - + + {% endif %} diff --git a/euscanwww/djeuscan/templates/euscan/_package_details.html b/euscanwww/djeuscan/templates/euscan/_package_details.html index 8746088..7f92133 100644 --- a/euscanwww/djeuscan/templates/euscan/_package_details.html +++ b/euscanwww/djeuscan/templates/euscan/_package_details.html @@ -33,12 +33,12 @@ {% for maintainer in package.maintainers.all %}

    {% if maintainer.name != maintainer.email %} - + {{ maintainer.name }} <{{ maintainer.email }}> {% else %} - + {{ maintainer.email }} {% endif %} diff --git a/euscanwww/djeuscan/templates/euscan/accounts/index.html b/euscanwww/djeuscan/templates/euscan/accounts/index.html index e9ac816..0e61ed6 100644 --- a/euscanwww/djeuscan/templates/euscan/accounts/index.html +++ b/euscanwww/djeuscan/templates/euscan/accounts/index.html @@ -75,7 +75,7 @@ Hot maintainers:

    diff --git a/euscanwww/djeuscan/templates/euscan/maintainer.html b/euscanwww/djeuscan/templates/euscan/maintainer.html index 65f498c..d73b582 100644 --- a/euscanwww/djeuscan/templates/euscan/maintainer.html +++ b/euscanwww/djeuscan/templates/euscan/maintainer.html @@ -11,7 +11,7 @@ {{ block.super }}
  • feed - + {{ maintainer.name|truncatechars:15 }}
    • @@ -22,10 +22,10 @@ Maintainer: {{ maintainer.name }} <{{ maintainer.email }}> {% if user.is_authenticated %} - - @@ -37,19 +37,19 @@

    Statistics

    Current statistics

    - pie versions - pie packages + pie versions + pie packages

    All Time

    - version weekly - versions monthly + version weekly + versions monthly

    - packages weekly - packages monthly + packages weekly + packages monthly