diff --git a/TODO b/TODO index 932c479..2f1799d 100644 --- a/TODO +++ b/TODO @@ -52,6 +52,9 @@ euscan euscanwww --------- +- Really fix mails +- Add an /about/config page that describe the current config (overlays, stuff in make.conf, euscan default settings, etc..) +- Create a rss for "scan world" + packages in GET - Always keep in db all found versions (when using an API only?). But don't display them if older than current packaged version, except maybe in the "upstream_version" column. ### packages diff --git a/bin/euscan b/bin/euscan index c17c5d3..255151b 100755 --- a/bin/euscan +++ b/bin/euscan @@ -292,9 +292,9 @@ def main(): from os.path import basename # To get the short name - output.error( + output.eerror( "The short ebuild name '%s' is ambiguous. Please specify" % - basename(pkgs[0]), + basename(pkgs[0]) + "one of the above fully-qualified ebuild names instead." ) exit_helper(1) diff --git a/euscanwww/djeuscan/api/handlers.py b/euscanwww/djeuscan/api/handlers.py index 18e1aca..922fe16 100644 --- a/euscanwww/djeuscan/api/handlers.py +++ b/euscanwww/djeuscan/api/handlers.py @@ -97,8 +97,12 @@ class PackagesHandler(AnonymousBaseHandler): herd = Herd.objects.get(herd=kwargs['herd']) packages = Package.objects.for_herd(herd, last_versions=True) data = {'herd': herd} - elif 'maintainer_id' in kwargs: - maintainer = Maintainer.objects.get(id=kwargs['maintainer_id']) + elif 'maintainer_id' in kwargs or 'maintainer_email' in kwargs: + if 'maintainer_id' in kwargs: + maintainer = Maintainer.objects.get(id=kwargs['maintainer_id']) + elif 'maintainer_email' in kwargs: + maintainer = Maintainer.objects.get(email=kwargs['maintainer_email']) + packages = Package.objects.for_maintainer( maintainer, last_versions=True diff --git a/euscanwww/djeuscan/api/urls.py b/euscanwww/djeuscan/api/urls.py index ffab482..578a8be 100644 --- a/euscanwww/djeuscan/api/urls.py +++ b/euscanwww/djeuscan/api/urls.py @@ -24,6 +24,8 @@ urlpatterns = patterns('djeuscan.api.views', url(r'^1.0/packages/by-maintainer/(?P\d+)\.(?P.+)$', packages_handler, name='api.views.packages'), + url(r'^1.0/packages/by-maintainer/(?P[^/]+)\.(?P.+)$', + packages_handler, name='api.views.packages'), url(r'^1.0/packages/by-herd/(?P[\@\{\}\w+.-]*)\.(?P.+)?$', packages_handler, name='api.views.packages'), url(r'^1.0/packages/by-category/(?P[\w+][\w+.-]*)\.(?P.+)?$', diff --git a/euscanwww/djeuscan/feeds.py b/euscanwww/djeuscan/feeds.py index 7399618..98590a9 100644 --- a/euscanwww/djeuscan/feeds.py +++ b/euscanwww/djeuscan/feeds.py @@ -146,9 +146,16 @@ class PackageFeed(BaseFeed): class MaintainerFeed(BaseFeed): feed_type = Atom1Feed - def get_object(self, request, maintainer_id): + def get_object(self, request, maintainer_id=None, maintainer_email=None): + print maintainer_id, maintainer_email + if maintainer_id: + obj = get_object_or_404(Maintainer, id=maintainer_id) + else: + obj = get_object_or_404(Maintainer, email=maintainer_email) + + print obj return { - "obj": get_object_or_404(Maintainer, id=maintainer_id), + "obj": obj, "options": request.GET, } @@ -159,8 +166,7 @@ class MaintainerFeed(BaseFeed): return "Last changes for %s" % data["obj"] def link(self, data): - return reverse('djeuscan.views.maintainer', - kwargs={'maintainer_id': data["obj"].id}) + return reverse('djeuscan.views.maintainer', args=[data["obj"].email]) def _items(self, data): return VersionLog.objects.for_maintainer(data["obj"]), 50 diff --git a/euscanwww/djeuscan/processing/misc/update_counters.py b/euscanwww/djeuscan/processing/misc/update_counters.py index fdbea9c..392b7b5 100644 --- a/euscanwww/djeuscan/processing/misc/update_counters.py +++ b/euscanwww/djeuscan/processing/misc/update_counters.py @@ -114,7 +114,6 @@ def update_counters(fast=False, nolog=False, logger=None): package.last_version_upstream_id = last_versions_upstream.get( package.id, default )['id'] - package.save() n_packages_gentoo = int(package.n_packaged == package.n_versions) diff --git a/euscanwww/djeuscan/processing/scan/scan_portage.py b/euscanwww/djeuscan/processing/scan/scan_portage.py index 1ce7197..1d6c866 100644 --- a/euscanwww/djeuscan/processing/scan/scan_portage.py +++ b/euscanwww/djeuscan/processing/scan/scan_portage.py @@ -13,7 +13,6 @@ from django.core.management.color import color_style from euscan.version import get_version_type from djeuscan.processing import FakeLogger -from djeuscan.processing.scan.scan_upstream import scan_upstream from djeuscan.models import Package, Version, VersionLog, Category, Overlay @@ -30,6 +29,10 @@ class ScanPortage(object): self._cache = {'packages': {}, 'versions': {}} self._overlays = None + self._updated_packages = set() + + def updated_packages(self): + return list(self._updated_packages) def cache_hash_package(self, category, name): return '%s/%s' % (category, name) @@ -200,13 +203,14 @@ class ScanPortage(object): packages_alive.add("%s/%s" % (cat, pkg)) new_version = False for cpv, slot, overlay, overlay_path in data['versions']: - new_version = new_version or self.store_version( + obj, created = self.store_version( package, cpv, slot, overlay, overlay_path ) + new_version = created or new_version # If the package has at least one new version scan upstream for it - if new_version and self.upstream: - scan_upstream([package], self.purge_versions, self.logger) + if new_version: + self._updated_packages.add(package) self.purge_old_packages(current_packages, packages_alive) self.purge_old_versions() @@ -269,7 +273,7 @@ class ScanPortage(object): # nothing to do (note: it can't be an upstream version because # overlay can't be empty here) if not created: - return False + return obj, created # New version created self.logger.info('+ [v] %s' % (obj)) @@ -292,7 +296,7 @@ class ScanPortage(object): vtype=obj.vtype, ) - return True + return obj, created def purge_old_packages(self, packages, alive): if not self.purge_packages: @@ -349,8 +353,7 @@ def scan_portage(packages=None, category=None, no_log=False, upstream=False, logger=logger, no_log=no_log, purge_packages=purge_packages, - purge_versions=purge_versions, - upstream=upstream + purge_versions=purge_versions ) logger.info('Scanning portage tree...') @@ -368,8 +371,10 @@ def scan_portage(packages=None, category=None, no_log=False, upstream=False, scan_handler.cache_store_version(version) logger.info('done') - if not packages: + if not packages and category: scan_handler.scan(category=category) + elif not packages: + scan_handler.scan() else: for pkg in packages: if isinstance(pkg, Package): @@ -394,3 +399,4 @@ def scan_portage(packages=None, category=None, no_log=False, upstream=False, logger.info("+ [o] %s", overlay["overlay"]) logger.info('Done.') + return scan_handler.updated_packages() diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py index d3c1b5b..769e4af 100644 --- a/euscanwww/djeuscan/tasks.py +++ b/euscanwww/djeuscan/tasks.py @@ -9,7 +9,6 @@ from celery.task import task, group #import portage from django.conf import settings -from django.core.cache import cache from django.template.loader import render_to_string from django.core.mail import send_mail from django.db.models import Q @@ -80,7 +79,6 @@ def update_counters(fast=False): """ Updates counters """ - logger = update_counters.get_logger() logger.info("Updating counters (fast=%s)...", fast) misc.update_counters(fast=fast) @@ -114,7 +112,7 @@ def scan_metadata(packages=[], category=None, populate=False): @task -def scan_portage(packages=[], category=None, upstream=True, +def scan_portage(packages=None, category=None, no_log=False, purge_packages=False, purge_versions=False, prefetch=False): """ @@ -131,17 +129,15 @@ def scan_portage(packages=[], category=None, upstream=True, else: logger.info("Starting portage scan...") - scan.scan_portage( + return scan.scan_portage( packages=packages, category=category, - upstream=upstream, no_log=no_log, purge_packages=purge_packages, purge_versions=purge_versions, prefetch=prefetch, logger=logger, ) - return True @task @@ -181,9 +177,17 @@ def update_portage(packages=None): # Workaround for celery bug when chaining groups update_portage_trees() - scan_portage(packages=[], purge_packages=True, purge_versions=True, - prefetch=True) - scan_metadata(packages=[], populate=True) + updated_packages = scan_portage( + packages=None, + purge_packages=True, + purge_versions=True, + prefetch=True + ) + scan_metadata(packages=None, populate=True) + if updated_packages: + group_chunks(scan_upstream, updated_packages, + settings.TASKS_UPSTREAM_GROUPS, + purge_versions=True)() update_counters(fast=False) """ Currently broken @@ -236,20 +240,12 @@ def scan_package_user(package): return True -@task +@task(rate_limit="1/m") def consume_refresh_queue(locked=False): """ Satisfies user requests for package refreshing, runs every minute """ - LOCK_ID = 'lock-consume-refresh-queue' - unlock = lambda: cache.delete(LOCK_ID) - lock = lambda: cache.add(LOCK_ID, True, 120) - logger = consume_refresh_queue.get_logger() - - if not locked and not lock(): - return - logger.info('Consuming package refresh request queue...') try: @@ -259,16 +255,14 @@ def consume_refresh_queue(locked=False): scan_package_user.delay(pkg) logger.info('Selected: %s' % pkg) except IndexError: - pass - finally: - unlock() + return if RefreshPackageQuery.objects.count(): logger.info('Restarting myself in 60s') - lock() consume_refresh_queue.apply_async( kwargs={'locked': True}, countdown=60 ) + return True @task(max_retries=10, default_retry_delay=10 * 60) @@ -280,6 +274,7 @@ def send_user_email(address, subject, text): ) except Exception, exc: raise send_user_email.retry(exc=exc) + return True @task @@ -322,6 +317,7 @@ def process_emails(profiles, only_if_vlogs=False): profile.last_email = now profile.save(force_update=True) + return True @task @@ -336,6 +332,7 @@ def send_update_email(): settings.TASKS_EMAIL_GROUPS, only_if_vlogs=True )() + return True @task @@ -345,6 +342,7 @@ def send_weekly_email(): email_activated=True ) group_chunks(process_emails, profiles, settings.TASKS_EMAIL_GROUPS)() + return True @task @@ -354,7 +352,7 @@ def send_monthly_email(): email_activated=True ) group_chunks(process_emails, profiles, settings.TASKS_EMAIL_GROUPS)() - + return True admin_tasks = [ regen_rrds, diff --git a/euscanwww/djeuscan/templates/euscan/_maintainers_table.html b/euscanwww/djeuscan/templates/euscan/_maintainers_table.html index bfbdd21..665e3e8 100644 --- a/euscanwww/djeuscan/templates/euscan/_maintainers_table.html +++ b/euscanwww/djeuscan/templates/euscan/_maintainers_table.html @@ -18,7 +18,7 @@ {% for maintainer in maintainers %} - + {% if maintainer.maintainers__name != maintainer.maintainers__email %} {{ maintainer.maintainers__name }} <{{ maintainer.maintainers__email }}> {% else %} @@ -31,8 +31,8 @@ {% if extras %} - - + + {% endif %} diff --git a/euscanwww/djeuscan/templates/euscan/_package_details.html b/euscanwww/djeuscan/templates/euscan/_package_details.html index 8746088..7f92133 100644 --- a/euscanwww/djeuscan/templates/euscan/_package_details.html +++ b/euscanwww/djeuscan/templates/euscan/_package_details.html @@ -33,12 +33,12 @@ {% for maintainer in package.maintainers.all %}

{% if maintainer.name != maintainer.email %} - + {{ maintainer.name }} <{{ maintainer.email }}> {% else %} - + {{ maintainer.email }} {% endif %} diff --git a/euscanwww/djeuscan/templates/euscan/accounts/index.html b/euscanwww/djeuscan/templates/euscan/accounts/index.html index e9ac816..0e61ed6 100644 --- a/euscanwww/djeuscan/templates/euscan/accounts/index.html +++ b/euscanwww/djeuscan/templates/euscan/accounts/index.html @@ -75,7 +75,7 @@ Hot maintainers:

diff --git a/euscanwww/djeuscan/templates/euscan/maintainer.html b/euscanwww/djeuscan/templates/euscan/maintainer.html index 65f498c..d73b582 100644 --- a/euscanwww/djeuscan/templates/euscan/maintainer.html +++ b/euscanwww/djeuscan/templates/euscan/maintainer.html @@ -11,7 +11,7 @@ {{ block.super }}
  • feed - + {{ maintainer.name|truncatechars:15 }}
  • @@ -22,10 +22,10 @@ Maintainer: {{ maintainer.name }} <{{ maintainer.email }}> {% if user.is_authenticated %} - - @@ -37,19 +37,19 @@

    Statistics

    Current statistics

    - pie versions - pie packages + pie versions + pie packages

    All Time

    - version weekly - versions monthly + version weekly + versions monthly

    - packages weekly - packages monthly + packages weekly + packages monthly